diff --git a/.gitignore b/.gitignore index f09b885358..fb0b654e32 100644 --- a/.gitignore +++ b/.gitignore @@ -450,3 +450,5 @@ __pycache__/ /apiserver.local.config e2e.namespace minikube.kubeconfig +apiserver.crt +apiserver.key diff --git a/cmd/package-server/apiserver.local.config/certificates/apiserver.crt b/cmd/package-server/apiserver.local.config/certificates/apiserver.crt deleted file mode 100644 index 878fa6ae1b..0000000000 --- a/cmd/package-server/apiserver.local.config/certificates/apiserver.crt +++ /dev/null @@ -1,37 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDGDCCAgCgAwIBAgIBAjANBgkqhkiG9w0BAQsFADAiMSAwHgYDVQQDDBdsb2Nh -bGhvc3QtY2FAMTU1MjQyMzAyMDAeFw0xOTAzMTIyMDM3MDFaFw0yMDAzMTEyMDM3 -MDFaMB8xHTAbBgNVBAMMFGxvY2FsaG9zdEAxNTUyNDIzMDIxMIIBIjANBgkqhkiG -9w0BAQEFAAOCAQ8AMIIBCgKCAQEAubkXRqN2xYxJiVhMjHnOtPCkU44QcLosVpIj -tbUgzjJt0BDv/XNCMhbpD3dfKjMKZiKXt1dKDK2Tl52AceWqipVQlCf7kiX+CjuO -gTAIEbVC7FWdu/sDI8BWbhs5knT+8Y7a5uGVexclZifvcbASuVtedLH47XI25Ak4 -s103Usy5Z2WXOLd79w/tsAr1kvQzveIdbn+upMu4to2wmfXhiLaU2qMhGoz+2hzm -z+SXkB7uCgFbGuLIUj99/faSZ3CAH6EwPIerAKtY+1hdVmsjqpIrSs4jD7YyfmVN -3+/MLTSMyHrghHYKt/SiRdCuVrbMhCylU8NFry+iuBIsOA202QIDAQABo1wwWjAO -BgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIw -ADAlBgNVHREEHjAcgglsb2NhbGhvc3SCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG -9w0BAQsFAAOCAQEAacr9G8nNsHQpLCW+0meGmDz9deTfLYldFCbCjsPiUDWs9tUn -O+04ykac2tEqZt2Ovkp6gntRPBCOKpgwHYvo0CJtCaL4yh6wYMvlbjHmHR/y+Ioy -HymMmaQ06iVIhb2KoKFJvFtFUVNg6QE9w7dm9/C73eHcv3JhqYhGw3qBfUI6lmIc -lWGj6WGVNfslofTYMkshbRGNZ3gFGkvcQvPOhKb/K4A3X9ZTGy9XyydVAOpdk/5n -FBD4gOJJVSq2jJ5SOTJd5Z/YrY2tbCfZeuuPuxBK4XG3hnLN2fk9URwfCDc9EUQg -aYagxskTB6jaDkFD5lfXxEc3W+/mP62i7mH/fQ== ------END CERTIFICATE----- ------BEGIN CERTIFICATE----- -MIIC4jCCAcqgAwIBAgIBATANBgkqhkiG9w0BAQsFADAiMSAwHgYDVQQDDBdsb2Nh -bGhvc3QtY2FAMTU1MjQyMzAyMDAeFw0xOTAzMTIyMDM3MDBaFw0yMDAzMTEyMDM3 -MDBaMCIxIDAeBgNVBAMMF2xvY2FsaG9zdC1jYUAxNTUyNDIzMDIwMIIBIjANBgkq -hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnQb0E1iZ/R1J8bdzDP/EFx73JpU6fw6T -aTY9QTWWgt4EcamLpJK5Z+dOLhj/i6rQbe/vKpI6BbBo+S6MuBemyUbc4VpoTde6 -Hn26uWSlkQA72GLHYWvD+ahdRpLxOFddog9xcfEoYN/rlpwMp030y6clQhrb4WML -x1uQzqyOvzRHAN4NqxmLXbepTyWqiM3tLe2f4mPfcg/vhwQ5TSqR/Rm3FPh3rDdA -zvk9bGkvyX8iAUoLw/0aHe2dzTfnvBvkTJFEaLq61FLQ/zfMVRhPI2Fwljxq+jSq -FoYju/vr1sWxKc+AFxDdAZdRey2Afi1bVf8JHiDU8FSe9UcfqBUoyQIDAQABoyMw -ITAOBgNVHQ8BAf8EBAMCAqQwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsF -AAOCAQEAmrIS4kJNVjKj4vSj0lNWzOjk31CI26rKwPo+cFhvnPh+eg6wI+3I/gLC -yf9X5KIPaNS5MGzNEmpr7Ml7IviqUn8rSoVryoQwKtqnMhsGr3/Y/Rrd27OIYEW+ -6/phRyI2rM8Vzo0RVdqcQT+6qvknbZ4fr/3Or3YbjycyfqNeL0SzXff+c8s9skDw -r9OV5uMvmVJv3VNBhAEX83I4zJsfrH9XtAmz255aw24vBGMUHYEdH15K/IBxh4LZ -Y5AXZhVazjlzwWwnUpu8k88vesCUay8c4VtXfXHQTk/oS/ZDn7eQ7hTvzqYfEH2k -znJYRthnuUZo6M/rtMWzXK6QuunRtg== ------END CERTIFICATE----- diff --git a/cmd/package-server/apiserver.local.config/certificates/apiserver.key b/cmd/package-server/apiserver.local.config/certificates/apiserver.key deleted file mode 100644 index 783945ec5f..0000000000 --- a/cmd/package-server/apiserver.local.config/certificates/apiserver.key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEAubkXRqN2xYxJiVhMjHnOtPCkU44QcLosVpIjtbUgzjJt0BDv -/XNCMhbpD3dfKjMKZiKXt1dKDK2Tl52AceWqipVQlCf7kiX+CjuOgTAIEbVC7FWd -u/sDI8BWbhs5knT+8Y7a5uGVexclZifvcbASuVtedLH47XI25Ak4s103Usy5Z2WX -OLd79w/tsAr1kvQzveIdbn+upMu4to2wmfXhiLaU2qMhGoz+2hzmz+SXkB7uCgFb -GuLIUj99/faSZ3CAH6EwPIerAKtY+1hdVmsjqpIrSs4jD7YyfmVN3+/MLTSMyHrg -hHYKt/SiRdCuVrbMhCylU8NFry+iuBIsOA202QIDAQABAoIBAEqc4o39c+TvdEea -Ur6I3RNyLgJna5FuKgvpkDEbAH/2YImblF7VZD2tWJpfEbtpX/8iXKNKjTREs6vQ -md6oLviX/hRXb8kKPGIuBRU/j65VjPpXdxQjRuKhDdgUVe/R0u6GvsjMzfnylZLR -7m9VFmCjJXJqYaA7J3Q7hC0DAQvhBiWk0lZHR7cjGeG37fIT2yzH7gf8M4VeYjCn -asatNUuAOORVfGudtKLCgFk/bmO1Nb5UwCYcz4OXVEpDBWrcg1SsvYwKxyUDxO8a -8A7TAWWEXjWK+sPmaJkUzRfnd/1chvlzcaawXfgfXRHcAaLWRaBu4fdYS7fwMYy6 -+/0Pa5ECgYEA0GCkaAl7qicfHTY6xTkBvJwkXu/rDIfzJCRVtdlXOhPmJ+F3+0Rj -0d+O6LMNSyJpYdOYeWOJbjHMJ92XIRJVxqF+K2O6dToEMTG2XbqMm2gtyn16BoTt -ngzcWqeo+zqwvHxLcM6L/tjivnbsI7mVDpdcBJZwVd6VwrR2NgRh0tUCgYEA5CsF -rJUlOR3JJ1CUTrT1G4smBES00lL3QFlhkiF4zWOW6NwhswZlYPkzqe6tgxmtGAuQ -mJINMcqWUkU18BWLh8RRTH+oKcUbmZkTqP9k/bqe6foIm8UyxVsSF80S4tRtMcWm -87Nd2h+FbYY2MP9RFscdDDd5FHf+weSCbnn0s/UCgYEAz05WQeqtTSp+meFJtsxw -HeR5irnFbkIScvJzEueXEACcCTEW3LO9Wx6+XmND5mvly51nI90S7L4+Das2n4BO -Nb6UdzZQWi/N2+NJOxZMrI+Ifts2eyXkAElrMAV85/QLwHkn1KKoRHIhortNUn1e -/ZU3xpikScmX1I0UzciuScECgYAbWrEOdL8GrvR7uyRcn0M3byI6psYK5RlxZIXX -EB48eXERL7r2jJDA5H92IwA4VG61EEXglLnyOzh0WonR47NbroSUqEVP5KqfaoO5 -4gyIgsQkhu5bRnQExxtPMS3Pdeo1al3On7Vjvh2v+MQscZ+WHH72BPyGILCxLCUa -+5IDtQKBgGE1Wl2dmdAyedzCX93oOjnVQ2xdH4s+4k7yHBYEt9AIzbuZCSZLMsf+ -hDoU/TokDRXkrHnRvZvhpljgjRJULktnmZxRWW8e/YXrp+gTvSq7/bZCob8Dgs80 -w21YuIgo6sXV2uvqGUbZ3YvJQU0GnoFB/GztGlmuVyU0jpsJKq5P ------END RSA PRIVATE KEY----- diff --git a/cmd/package-server/main.go b/cmd/package-server/main.go index a489cc6ab4..e41353069a 100644 --- a/cmd/package-server/main.go +++ b/cmd/package-server/main.go @@ -3,58 +3,25 @@ package main import ( "flag" "os" - "time" log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" "k8s.io/component-base/logs" - "github.com/operator-framework/operator-lifecycle-manager/pkg/package-server/server" "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/signals" + "github.com/operator-framework/operator-lifecycle-manager/pkg/package-server/server" ) -const ( - defaultWakeupInterval = 5 * time.Minute -) - -// config flags defined globally so that they appear on the test binary as well -var ( - ctx = signals.Context() - options = server.NewPackageServerOptions(os.Stdout, os.Stderr) - cmd = &cobra.Command{ - Short: "Launch a package-server", - Long: "Launch a package-server", - RunE: func(c *cobra.Command, args []string) error { - if err := options.Run(ctx); err != nil { - return err - } - return nil - }, - } -) - -func init() { - flags := cmd.Flags() - - flags.DurationVar(&options.WakeupInterval, "interval", options.WakeupInterval, "Interval at which to re-sync CatalogSources") - flags.StringVar(&options.GlobalNamespace, "global-namespace", options.GlobalNamespace, "Name of the namespace where the global CatalogSources are located") - flags.StringSliceVar(&options.WatchedNamespaces, "watched-namespaces", options.WatchedNamespaces, "List of namespaces the package-server will watch watch for CatalogSources") - flags.StringVar(&options.Kubeconfig, "kubeconfig", options.Kubeconfig, "The path to the kubeconfig used to connect to the Kubernetes API server and the Kubelets (defaults to in-cluster config)") - flags.BoolVar(&options.Debug, "debug", options.Debug, "use debug log level") - - options.SecureServing.AddFlags(flags) - options.Authentication.AddFlags(flags) - options.Authorization.AddFlags(flags) - options.Features.AddFlags(flags) - - flags.AddGoFlagSet(flag.CommandLine) - flags.Parse(flag.Args()) -} - func main() { logs.InitLogs() defer logs.FlushLogs() + ctx := signals.Context() + options := server.NewPackageServerOptions(os.Stdout, os.Stderr) + cmd := server.NewCommandStartPackageServer(ctx, options) + cmd.Flags().AddGoFlagSet(flag.CommandLine) + if err := cmd.Flags().Parse(flag.Args()); err != nil { + log.Fatal(err) + } if err := cmd.Execute(); err != nil { log.Fatal(err) } diff --git a/go.mod b/go.mod index 233c5043e1..4c61059ebb 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,6 @@ require ( github.com/fsnotify/fsnotify v1.4.7 github.com/ghodss/yaml v1.0.0 github.com/go-openapi/spec v0.19.2 - github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff // indirect github.com/golang/mock v1.3.1 github.com/google/btree v1.0.0 // indirect @@ -34,6 +33,7 @@ require ( github.com/stretchr/testify v1.4.0 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect + golang.org/x/sync v0.0.0-20190423024810-112230192c58 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 gonum.org/v1/gonum v0.0.0-20190710053202-4340aa3071a0 // indirect google.golang.org/grpc v1.23.0 @@ -44,6 +44,7 @@ require ( k8s.io/client-go v8.0.0+incompatible k8s.io/code-generator v0.0.0 k8s.io/component-base v0.0.0 + k8s.io/klog v0.4.0 k8s.io/kube-aggregator v0.0.0 k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf k8s.io/kubernetes v1.16.0 diff --git a/go.sum b/go.sum index 6736413775..661fb2582c 100644 --- a/go.sum +++ b/go.sum @@ -609,6 +609,7 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/pkg/lib/event/event.go b/pkg/lib/event/event.go index 79f73a0d9f..7aff735b85 100644 --- a/pkg/lib/event/event.go +++ b/pkg/lib/event/event.go @@ -1,10 +1,10 @@ package event import ( - "github.com/golang/glog" v1 "k8s.io/api/core/v1" typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/record" + "k8s.io/klog" "github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/clientset/versioned/scheme" ) @@ -15,7 +15,7 @@ const component string = "operator-lifecycle-manager" // used to post Events to different object's lifecycles. func NewRecorder(event typedcorev1.EventInterface) (record.EventRecorder, error) { eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: event}) recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: component}) diff --git a/pkg/lib/operatorclient/apiservice.go b/pkg/lib/operatorclient/apiservice.go index 76b4792dd8..475d2639b3 100644 --- a/pkg/lib/operatorclient/apiservice.go +++ b/pkg/lib/operatorclient/apiservice.go @@ -3,9 +3,9 @@ package operatorclient import ( "fmt" - "github.com/golang/glog" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" ) @@ -26,7 +26,7 @@ func (c *Client) DeleteAPIService(name string, options *metav1.DeleteOptions) er // UpdateAPIService will update the given APIService resource. func (c *Client) UpdateAPIService(apiService *apiregistrationv1.APIService) (*apiregistrationv1.APIService, error) { - glog.V(4).Infof("[UPDATE APIService]: %s", apiService.GetName()) + klog.V(4).Infof("[UPDATE APIService]: %s", apiService.GetName()) oldAPIService, err := c.GetAPIService(apiService.GetName()) if err != nil { return nil, err diff --git a/pkg/lib/operatorclient/clusterrole.go b/pkg/lib/operatorclient/clusterrole.go index ee736fd2fd..b2f9859e66 100644 --- a/pkg/lib/operatorclient/clusterrole.go +++ b/pkg/lib/operatorclient/clusterrole.go @@ -3,10 +3,10 @@ package operatorclient import ( "fmt" - "github.com/golang/glog" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" ) // CreateClusterRole creates the ClusterRole. @@ -26,7 +26,7 @@ func (c *Client) DeleteClusterRole(name string, options *metav1.DeleteOptions) e // UpdateClusterRole will update the given ClusterRole. func (c *Client) UpdateClusterRole(crb *rbacv1.ClusterRole) (*rbacv1.ClusterRole, error) { - glog.V(4).Infof("[UPDATE Role]: %s", crb.GetName()) + klog.V(4).Infof("[UPDATE Role]: %s", crb.GetName()) oldCrb, err := c.GetClusterRole(crb.GetName()) if err != nil { return nil, err diff --git a/pkg/lib/operatorclient/clusterrolebinding.go b/pkg/lib/operatorclient/clusterrolebinding.go index 30e6826768..494d216c4c 100755 --- a/pkg/lib/operatorclient/clusterrolebinding.go +++ b/pkg/lib/operatorclient/clusterrolebinding.go @@ -3,10 +3,10 @@ package operatorclient import ( "fmt" - "github.com/golang/glog" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" ) // CreateRoleBinding creates the roleBinding. @@ -26,7 +26,7 @@ func (c *Client) DeleteClusterRoleBinding(name string, options *metav1.DeleteOpt // UpdateRoleBinding will update the given RoleBinding resource. func (c *Client) UpdateClusterRoleBinding(crb *rbacv1.ClusterRoleBinding) (*rbacv1.ClusterRoleBinding, error) { - glog.V(4).Infof("[UPDATE RoleBinding]: %s", crb.GetName()) + klog.V(4).Infof("[UPDATE RoleBinding]: %s", crb.GetName()) oldCrb, err := c.GetClusterRoleBinding(crb.GetName()) if err != nil { return nil, err diff --git a/pkg/lib/operatorclient/customresources.go b/pkg/lib/operatorclient/customresources.go index 57201a4403..beda8ce20b 100644 --- a/pkg/lib/operatorclient/customresources.go +++ b/pkg/lib/operatorclient/customresources.go @@ -7,11 +7,11 @@ import ( "strings" "time" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog" ) // CustomResourceList represents a list of custom resource objects that will @@ -25,7 +25,7 @@ type CustomResourceList struct { // GetCustomResource returns the custom resource as *unstructured.Unstructured by the given name. func (c *Client) GetCustomResource(apiGroup, version, namespace, resourcePlural, resourceName string) (*unstructured.Unstructured, error) { - glog.V(4).Infof("[GET CUSTOM RESOURCE]: %s:%s", namespace, resourceName) + klog.V(4).Infof("[GET CUSTOM RESOURCE]: %s:%s", namespace, resourceName) var object unstructured.Unstructured b, err := c.GetCustomResourceRaw(apiGroup, version, namespace, resourcePlural, resourceName) @@ -41,17 +41,17 @@ func (c *Client) GetCustomResource(apiGroup, version, namespace, resourcePlural, // GetCustomResourceRaw returns the custom resource's raw body data by the given name. func (c *Client) GetCustomResourceRaw(apiGroup, version, namespace, resourcePlural, resourceName string) ([]byte, error) { - glog.V(4).Infof("[GET CUSTOM RESOURCE RAW]: %s:%s", namespace, resourceName) + klog.V(4).Infof("[GET CUSTOM RESOURCE RAW]: %s:%s", namespace, resourceName) httpRestClient := c.extInterface.ApiextensionsV1beta1().RESTClient() uri := customResourceURI(apiGroup, version, namespace, resourcePlural, resourceName) - glog.V(4).Infof("[GET]: %s", uri) + klog.V(4).Infof("[GET]: %s", uri) return httpRestClient.Get().RequestURI(uri).DoRaw() } // CreateCustomResource creates the custom resource. func (c *Client) CreateCustomResource(item *unstructured.Unstructured) error { - glog.V(4).Infof("[CREATE CUSTOM RESOURCE]: %s:%s", item.GetNamespace(), item.GetName()) + klog.V(4).Infof("[CREATE CUSTOM RESOURCE]: %s:%s", item.GetNamespace(), item.GetName()) kind := item.GetKind() namespace := item.GetNamespace() apiVersion := item.GetAPIVersion() @@ -70,12 +70,12 @@ func (c *Client) CreateCustomResource(item *unstructured.Unstructured) error { // CreateCustomResourceRaw creates the raw bytes of the custom resource. func (c *Client) CreateCustomResourceRaw(apiGroup, version, namespace, kind string, data []byte) error { - glog.V(4).Infof("[CREATE CUSTOM RESOURCE RAW]: %s:%s", namespace, kind) + klog.V(4).Infof("[CREATE CUSTOM RESOURCE RAW]: %s:%s", namespace, kind) var statusCode int httpRestClient := c.extInterface.ApiextensionsV1beta1().RESTClient() uri := customResourceDefinitionURI(apiGroup, version, namespace, kind) - glog.V(4).Infof("[POST]: %s", uri) + klog.V(4).Infof("[POST]: %s", uri) result := httpRestClient.Post().RequestURI(uri).Body(data).Do() if result.Error() != nil { @@ -83,7 +83,7 @@ func (c *Client) CreateCustomResourceRaw(apiGroup, version, namespace, kind stri } result.StatusCode(&statusCode) - glog.V(4).Infof("Written %s, status: %d", uri, statusCode) + klog.V(4).Infof("Written %s, status: %d", uri, statusCode) if statusCode != 201 { return fmt.Errorf("unexpected status code %d, expecting 201", statusCode) @@ -94,7 +94,7 @@ func (c *Client) CreateCustomResourceRaw(apiGroup, version, namespace, kind stri // CreateCustomResourceRawIfNotFound creates the raw bytes of the custom resource if it doesn't exist. // It also returns a boolean to indicate whether a new custom resource is created. func (c *Client) CreateCustomResourceRawIfNotFound(apiGroup, version, namespace, kind, name string, data []byte) (bool, error) { - glog.V(4).Infof("[CREATE CUSTOM RESOURCE RAW if not found]: %s:%s", namespace, name) + klog.V(4).Infof("[CREATE CUSTOM RESOURCE RAW if not found]: %s:%s", namespace, name) _, err := c.GetCustomResource(apiGroup, version, namespace, kind, name) if err == nil { return false, nil @@ -112,7 +112,7 @@ func (c *Client) CreateCustomResourceRawIfNotFound(apiGroup, version, namespace, // UpdateCustomResource updates the custom resource. // To do an atomic update, use AtomicModifyCustomResource(). func (c *Client) UpdateCustomResource(item *unstructured.Unstructured) error { - glog.V(4).Infof("[UPDATE CUSTOM RESOURCE]: %s:%s", item.GetNamespace(), item.GetName()) + klog.V(4).Infof("[UPDATE CUSTOM RESOURCE]: %s:%s", item.GetNamespace(), item.GetName()) kind := item.GetKind() name := item.GetName() namespace := item.GetNamespace() @@ -132,12 +132,12 @@ func (c *Client) UpdateCustomResource(item *unstructured.Unstructured) error { // UpdateCustomResourceRaw updates the thirdparty resource with the raw data. func (c *Client) UpdateCustomResourceRaw(apiGroup, version, namespace, resourcePlural, resourceName string, data []byte) error { - glog.V(4).Infof("[UPDATE CUSTOM RESOURCE RAW]: %s:%s", namespace, resourceName) + klog.V(4).Infof("[UPDATE CUSTOM RESOURCE RAW]: %s:%s", namespace, resourceName) var statusCode int httpRestClient := c.extInterface.ApiextensionsV1beta1().RESTClient() uri := customResourceURI(apiGroup, version, namespace, resourcePlural, resourceName) - glog.V(4).Infof("[PUT]: %s", uri) + klog.V(4).Infof("[PUT]: %s", uri) result := httpRestClient.Put().RequestURI(uri).Body(data).Do() if result.Error() != nil { @@ -145,7 +145,7 @@ func (c *Client) UpdateCustomResourceRaw(apiGroup, version, namespace, resourceP } result.StatusCode(&statusCode) - glog.V(4).Infof("Updated %s, status: %d", uri, statusCode) + klog.V(4).Infof("Updated %s, status: %d", uri, statusCode) if statusCode != 200 { return fmt.Errorf("unexpected status code %d, expecting 200", statusCode) @@ -156,7 +156,7 @@ func (c *Client) UpdateCustomResourceRaw(apiGroup, version, namespace, resourceP // CreateOrUpdateCustomeResourceRaw creates the custom resource if it doesn't exist. // If the custom resource exists, it updates the existing one. func (c *Client) CreateOrUpdateCustomeResourceRaw(apiGroup, version, namespace, resourcePlural, resourceName string, data []byte) error { - glog.V(4).Infof("[CREATE OR UPDATE UPDATE CUSTOM RESOURCE RAW]: %s:%s", namespace, resourceName) + klog.V(4).Infof("[CREATE OR UPDATE UPDATE CUSTOM RESOURCE RAW]: %s:%s", namespace, resourceName) old, err := c.GetCustomResourceRaw(apiGroup, version, namespace, resourcePlural, resourceName) if err != nil { if !errors.IsNotFound(err) { @@ -186,11 +186,11 @@ func (c *Client) CreateOrUpdateCustomeResourceRaw(apiGroup, version, namespace, // DeleteCustomResource deletes the with the given name. func (c *Client) DeleteCustomResource(apiGroup, version, namespace, resourcePlural, resourceName string) error { - glog.V(4).Infof("[DELETE CUSTOM RESOURCE]: %s:%s", namespace, resourceName) + klog.V(4).Infof("[DELETE CUSTOM RESOURCE]: %s:%s", namespace, resourceName) httpRestClient := c.extInterface.ApiextensionsV1beta1().RESTClient() uri := customResourceURI(apiGroup, version, namespace, resourcePlural, resourceName) - glog.V(4).Infof("[DELETE]: %s", uri) + klog.V(4).Infof("[DELETE]: %s", uri) _, err := httpRestClient.Delete().RequestURI(uri).DoRaw() return err } @@ -201,31 +201,31 @@ type CustomResourceModifier func(*unstructured.Unstructured, interface{}) error // AtomicModifyCustomResource gets the custom resource, modifies it and writes it back. // If it's modified by other writers, we will retry until it succeeds. func (c *Client) AtomicModifyCustomResource(apiGroup, version, namespace, resourcePlural, resourceName string, f CustomResourceModifier, data interface{}) error { - glog.V(4).Infof("[ATOMIC MODIFY CUSTOM RESOURCE]: %s:%s", namespace, resourceName) + klog.V(4).Infof("[ATOMIC MODIFY CUSTOM RESOURCE]: %s:%s", namespace, resourceName) return wait.PollInfinite(time.Second, func() (bool, error) { var customResource unstructured.Unstructured b, err := c.GetCustomResourceRaw(apiGroup, version, namespace, resourcePlural, resourceName) if err != nil { - glog.Errorf("Failed to get CUSTOM RESOURCE %q, kind:%q: %v", resourceName, resourcePlural, err) + klog.Errorf("Failed to get CUSTOM RESOURCE %q, kind:%q: %v", resourceName, resourcePlural, err) return false, err } if err := json.Unmarshal(b, &customResource); err != nil { - glog.Errorf("Failed to unmarshal CUSTOM RESOURCE %q, kind:%q: %v", resourceName, resourcePlural, err) + klog.Errorf("Failed to unmarshal CUSTOM RESOURCE %q, kind:%q: %v", resourceName, resourcePlural, err) return false, err } if err := f(&customResource, data); err != nil { - glog.Errorf("Failed to modify the CUSTOM RESOURCE %q, kind:%q: %v", resourceName, resourcePlural, err) + klog.Errorf("Failed to modify the CUSTOM RESOURCE %q, kind:%q: %v", resourceName, resourcePlural, err) return false, err } if err := c.UpdateCustomResource(&customResource); err != nil { if errors.IsConflict(err) { - glog.Errorf("Failed to update CUSTOM RESOURCE %q, kind:%q: %v, will retry", resourceName, resourcePlural, err) + klog.Errorf("Failed to update CUSTOM RESOURCE %q, kind:%q: %v, will retry", resourceName, resourcePlural, err) return false, nil } - glog.Errorf("Failed to update CUSTOM RESOURCE %q, kind:%q: %v", resourceName, resourcePlural, err) + klog.Errorf("Failed to update CUSTOM RESOURCE %q, kind:%q: %v", resourceName, resourcePlural, err) return false, err } @@ -273,13 +273,13 @@ func customResourceDefinitionURI(apiGroup, version, namespace, resourcePlural st // ListCustomResource lists all custom resources for the given namespace. func (c *Client) ListCustomResource(apiGroup, version, namespace, resourcePlural string) (*CustomResourceList, error) { - glog.V(4).Infof("LIST CUSTOM RESOURCE]: %s", resourcePlural) + klog.V(4).Infof("LIST CUSTOM RESOURCE]: %s", resourcePlural) var crList CustomResourceList httpRestClient := c.extInterface.ApiextensionsV1beta1().RESTClient() uri := customResourceDefinitionURI(apiGroup, version, namespace, resourcePlural) - glog.V(4).Infof("[GET]: %s", uri) + klog.V(4).Infof("[GET]: %s", uri) bytes, err := httpRestClient.Get().RequestURI(uri).DoRaw() if err != nil { return nil, fmt.Errorf("failed to get custom resource list: %v", err) diff --git a/pkg/lib/operatorclient/deployment.go b/pkg/lib/operatorclient/deployment.go index 6a62ca382b..f7dc22a20f 100644 --- a/pkg/lib/operatorclient/deployment.go +++ b/pkg/lib/operatorclient/deployment.go @@ -5,13 +5,13 @@ import ( "fmt" "time" - "github.com/golang/glog" appsv1 "k8s.io/api/apps/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog" ) const ( @@ -20,19 +20,19 @@ const ( // GetDeployment returns the Deployment object for the given namespace and name. func (c *Client) GetDeployment(namespace, name string) (*appsv1.Deployment, error) { - glog.V(4).Infof("[GET Deployment]: %s:%s", namespace, name) + klog.V(4).Infof("[GET Deployment]: %s:%s", namespace, name) return c.AppsV1().Deployments(namespace).Get(name, metav1.GetOptions{}) } // CreateDeployment creates the Deployment object. func (c *Client) CreateDeployment(dep *appsv1.Deployment) (*appsv1.Deployment, error) { - glog.V(4).Infof("[CREATE Deployment]: %s:%s", dep.Namespace, dep.Name) + klog.V(4).Infof("[CREATE Deployment]: %s:%s", dep.Namespace, dep.Name) return c.AppsV1().Deployments(dep.Namespace).Create(dep) } // DeleteDeployment deletes the Deployment object. func (c *Client) DeleteDeployment(namespace, name string, options *metav1.DeleteOptions) error { - glog.V(4).Infof("[DELETE Deployment]: %s:%s", namespace, name) + klog.V(4).Infof("[DELETE Deployment]: %s:%s", namespace, name) return c.AppsV1().Deployments(namespace).Delete(name, options) } @@ -50,7 +50,7 @@ func (c *Client) UpdateDeployment(dep *appsv1.Deployment) (*appsv1.Deployment, b // Returns the latest Deployment and true if it was updated, or an error. func (c *Client) PatchDeployment(original, modified *appsv1.Deployment) (*appsv1.Deployment, bool, error) { namespace, name := modified.Namespace, modified.Name - glog.V(4).Infof("[PATCH Deployment]: %s:%s", namespace, name) + klog.V(4).Infof("[PATCH Deployment]: %s:%s", namespace, name) current, err := c.AppsV1().Deployments(namespace).Get(name, metav1.GetOptions{}) if err != nil { @@ -86,7 +86,7 @@ func (c *Client) RollingUpdateDeployment(dep *appsv1.Deployment) (*appsv1.Deploy // RollingUpdateDeploymentMigrations will run any before / during / after migrations that have been // specified in the upgrade options. func (c *Client) RollingUpdateDeploymentMigrations(namespace, name string, f UpdateFunction) (*appsv1.Deployment, bool, error) { - glog.V(4).Infof("[ROLLING UPDATE Deployment]: %s:%s", namespace, name) + klog.V(4).Infof("[ROLLING UPDATE Deployment]: %s:%s", namespace, name) return c.RollingPatchDeploymentMigrations(namespace, name, updateToPatch(f)) } @@ -107,7 +107,7 @@ func (c *Client) RollingPatchDeployment(original, modified *appsv1.Deployment) ( // RollingPatchDeploymentMigrations will run any before / after migrations that have been specified // in the upgrade options. func (c *Client) RollingPatchDeploymentMigrations(namespace, name string, f PatchFunction) (*appsv1.Deployment, bool, error) { - glog.V(4).Infof("[ROLLING PATCH Deployment]: %s:%s", namespace, name) + klog.V(4).Infof("[ROLLING PATCH Deployment]: %s:%s", namespace, name) current, err := c.AppsV1().Deployments(namespace).Get(name, metav1.GetOptions{}) if err != nil { @@ -163,7 +163,7 @@ func (c *Client) waitForDeploymentRollout(dep *appsv1.Deployment) error { if err != nil { // Do not return error here, as we could be updating the API Server itself, in which case we // want to continue waiting. - glog.Errorf("error getting Deployment %s during rollout: %v", dep.Name, err) + klog.Errorf("error getting Deployment %s during rollout: %v", dep.Name, err) return false, nil } if d.Generation <= d.Status.ObservedGeneration && d.Status.UpdatedReplicas == d.Status.Replicas && d.Status.UnavailableReplicas == 0 { @@ -177,7 +177,7 @@ func (c *Client) waitForDeploymentRollout(dep *appsv1.Deployment) error { // already exists, it will update the Deployment and wait for it to rollout. Returns true if the // Deployment was created or updated, false if there was no update. func (c *Client) CreateOrRollingUpdateDeployment(dep *appsv1.Deployment) (*appsv1.Deployment, bool, error) { - glog.V(4).Infof("[CREATE OR ROLLING UPDATE Deployment]: %s:%s", dep.Namespace, dep.Name) + klog.V(4).Infof("[CREATE OR ROLLING UPDATE Deployment]: %s:%s", dep.Namespace, dep.Name) _, err := c.GetDeployment(dep.Namespace, dep.Name) if err != nil { @@ -196,7 +196,7 @@ func (c *Client) CreateOrRollingUpdateDeployment(dep *appsv1.Deployment) (*appsv // ListDeploymentsWithLabels returns a list of deployments that matches the label selector. // An empty list will be returned if no such deployments is found. func (c *Client) ListDeploymentsWithLabels(namespace string, labels labels.Set) (*appsv1.DeploymentList, error) { - glog.V(4).Infof("[LIST Deployments] in %s, labels: %v", namespace, labels) + klog.V(4).Infof("[LIST Deployments] in %s, labels: %v", namespace, labels) opts := metav1.ListOptions{LabelSelector: labels.String()} return c.AppsV1().Deployments(namespace).List(opts) diff --git a/pkg/lib/operatorclient/role.go b/pkg/lib/operatorclient/role.go index fc07d73ace..45a32ec593 100755 --- a/pkg/lib/operatorclient/role.go +++ b/pkg/lib/operatorclient/role.go @@ -3,10 +3,10 @@ package operatorclient import ( "fmt" - "github.com/golang/glog" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" ) // CreateRole creates the role. @@ -26,7 +26,7 @@ func (c *Client) DeleteRole(namespace, name string, options *metav1.DeleteOption // UpdateRole will update the given Role resource. func (c *Client) UpdateRole(crb *rbacv1.Role) (*rbacv1.Role, error) { - glog.V(4).Infof("[UPDATE Role]: %s", crb.GetName()) + klog.V(4).Infof("[UPDATE Role]: %s", crb.GetName()) oldCrb, err := c.GetRole(crb.GetNamespace(), crb.GetName()) if err != nil { return nil, err diff --git a/pkg/lib/operatorclient/rolebinding.go b/pkg/lib/operatorclient/rolebinding.go index e9d084e851..358b6ffe21 100755 --- a/pkg/lib/operatorclient/rolebinding.go +++ b/pkg/lib/operatorclient/rolebinding.go @@ -3,10 +3,10 @@ package operatorclient import ( "fmt" - "github.com/golang/glog" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" ) // CreateRoleBinding creates the roleBinding. @@ -26,7 +26,7 @@ func (c *Client) DeleteRoleBinding(namespace, name string, options *metav1.Delet // UpdateRoleBinding will update the given RoleBinding resource. func (c *Client) UpdateRoleBinding(crb *rbacv1.RoleBinding) (*rbacv1.RoleBinding, error) { - glog.V(4).Infof("[UPDATE RoleBinding]: %s", crb.GetName()) + klog.V(4).Infof("[UPDATE RoleBinding]: %s", crb.GetName()) oldCrb, err := c.GetRoleBinding(crb.GetNamespace(), crb.GetName()) if err != nil { return nil, err diff --git a/pkg/lib/operatorclient/secret.go b/pkg/lib/operatorclient/secret.go index e20d3d8b6f..5411385f27 100644 --- a/pkg/lib/operatorclient/secret.go +++ b/pkg/lib/operatorclient/secret.go @@ -3,10 +3,10 @@ package operatorclient import ( "fmt" - "github.com/golang/glog" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" ) // CreateSecret creates the Secret. @@ -26,7 +26,7 @@ func (c *Client) DeleteSecret(namespace, name string, options *metav1.DeleteOpti // UpdateSecret will update the given Secret resource. func (c *Client) UpdateSecret(secret *v1.Secret) (*v1.Secret, error) { - glog.V(4).Infof("[UPDATE Secret]: %s", secret.GetName()) + klog.V(4).Infof("[UPDATE Secret]: %s", secret.GetName()) oldSa, err := c.GetSecret(secret.GetNamespace(), secret.GetName()) if err != nil { return nil, err diff --git a/pkg/lib/operatorclient/service.go b/pkg/lib/operatorclient/service.go index 4491233d78..f14f21c886 100644 --- a/pkg/lib/operatorclient/service.go +++ b/pkg/lib/operatorclient/service.go @@ -3,10 +3,10 @@ package operatorclient import ( "fmt" - "github.com/golang/glog" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" ) // CreateService creates the Service. @@ -26,7 +26,7 @@ func (c *Client) DeleteService(namespace, name string, options *metav1.DeleteOpt // UpdateService will update the given Service resource. func (c *Client) UpdateService(service *v1.Service) (*v1.Service, error) { - glog.V(4).Infof("[UPDATE Service]: %s", service.GetName()) + klog.V(4).Infof("[UPDATE Service]: %s", service.GetName()) oldSa, err := c.GetService(service.GetNamespace(), service.GetName()) if err != nil { return nil, err diff --git a/pkg/lib/operatorclient/serviceaccount.go b/pkg/lib/operatorclient/serviceaccount.go index 8af9c5d2d0..1d16e8f491 100644 --- a/pkg/lib/operatorclient/serviceaccount.go +++ b/pkg/lib/operatorclient/serviceaccount.go @@ -3,10 +3,10 @@ package operatorclient import ( "fmt" - "github.com/golang/glog" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" ) // CreateServiceAccount creates the serviceAccount. @@ -26,7 +26,7 @@ func (c *Client) DeleteServiceAccount(namespace, name string, options *metav1.De // UpdateServiceAccount will update the given ServiceAccount resource. func (c *Client) UpdateServiceAccount(sa *v1.ServiceAccount) (*v1.ServiceAccount, error) { - glog.V(4).Infof("[UPDATE ServiceAccount]: %s", sa.GetName()) + klog.V(4).Infof("[UPDATE ServiceAccount]: %s", sa.GetName()) oldSa, err := c.GetServiceAccount(sa.GetNamespace(), sa.GetName()) if err != nil { return nil, err diff --git a/pkg/package-server/provider/registry.go b/pkg/package-server/provider/registry.go index c09c64885e..f96a77a841 100644 --- a/pkg/package-server/provider/registry.go +++ b/pkg/package-server/provider/registry.go @@ -13,105 +13,131 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/connectivity" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + utilerrors "k8s.io/apimachinery/pkg/util/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/tools/cache" - "k8s.io/kubernetes/pkg/util/labels" + utillabels "k8s.io/kubernetes/pkg/util/labels" operatorsv1alpha1 "github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1" "github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/clientset/versioned" "github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/informers/externalversions" + operatorslisters "github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/listers/operators/v1alpha1" + registrygrpc "github.com/operator-framework/operator-lifecycle-manager/pkg/controller/registry/grpc" + "github.com/operator-framework/operator-lifecycle-manager/pkg/controller/registry/resolver" "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/queueinformer" "github.com/operator-framework/operator-lifecycle-manager/pkg/package-server/apis/operators" + pkglisters "github.com/operator-framework/operator-lifecycle-manager/pkg/package-server/client/listers/operators/internalversion" ) const ( - defaultConnectionTimeout = 5 * time.Second + catalogIndex = "catalog" + cacheTimeout = 5 * time.Minute + readyTimeout = 10 * time.Minute + stateTimeout = 20 * time.Second ) -type sourceKey struct { - name string - namespace string +func getSourceKey(pkg *operators.PackageManifest) (key *resolver.CatalogKey) { + if pkg != nil { + key = &resolver.CatalogKey{ + Namespace: pkg.Status.CatalogSourceNamespace, + Name: pkg.Status.CatalogSource, + } + } + + return +} + +func catalogIndexFunc(obj interface{}) ([]string, error) { + pkg, ok := obj.(*operators.PackageManifest) + if !ok { + return []string{""}, fmt.Errorf("obj is not a packagemanifest %v", obj) + } + + return []string{getSourceKey(pkg).String()}, nil } type registryClient struct { api.RegistryClient - source *operatorsv1alpha1.CatalogSource + catsrc *operatorsv1alpha1.CatalogSource conn *grpc.ClientConn } -func newRegistryClient(source *operatorsv1alpha1.CatalogSource, conn *grpc.ClientConn) registryClient { - return registryClient{ +func newRegistryClient(catsrc *operatorsv1alpha1.CatalogSource, conn *grpc.ClientConn) *registryClient { + return ®istryClient{ RegistryClient: api.NewRegistryClient(conn), - source: source, + catsrc: catsrc, conn: conn, } } +func (r *registryClient) key() (key resolver.CatalogKey, err error) { + if r.catsrc == nil { + err = fmt.Errorf("cannot get key, nil catalog") + return + } + + key = resolver.CatalogKey{ + Namespace: r.catsrc.GetNamespace(), + Name: r.catsrc.GetName(), + } + + return +} + // RegistryProvider aggregates several `CatalogSources` and establishes gRPC connections to their registry servers. type RegistryProvider struct { queueinformer.Operator + runOnce sync.Once - mu sync.RWMutex globalNamespace string - clients map[sourceKey]registryClient + sources *registrygrpc.SourceStore + cache cache.Indexer + pkgLister pkglisters.PackageManifestLister + catsrcLister operatorslisters.CatalogSourceLister } var _ PackageManifestProvider = &RegistryProvider{} -func NewRegistryProvider(ctx context.Context, crClient versioned.Interface, operator queueinformer.Operator, wakeupInterval time.Duration, watchedNamespaces []string, globalNamespace string) (*RegistryProvider, error) { +func NewRegistryProvider(ctx context.Context, crClient versioned.Interface, operator queueinformer.Operator, wakeupInterval time.Duration, globalNamespace string) (*RegistryProvider, error) { p := &RegistryProvider{ Operator: operator, globalNamespace: globalNamespace, - clients: make(map[sourceKey]registryClient), + cache: cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{ + cache.NamespaceIndex: cache.MetaNamespaceIndexFunc, + catalogIndex: catalogIndexFunc, + }), } - - for _, namespace := range watchedNamespaces { - informerFactory := externalversions.NewSharedInformerFactoryWithOptions(crClient, wakeupInterval, externalversions.WithNamespace(namespace)) - catsrcInformer := informerFactory.Operators().V1alpha1().CatalogSources() - - // Register queue and QueueInformer - logrus.WithField("namespace", namespace).Info("watching catalogsources") - catsrcQueueInformer, err := queueinformer.NewQueueInformer( - ctx, - queueinformer.WithInformer(catsrcInformer.Informer()), - queueinformer.WithSyncer(queueinformer.LegacySyncHandler(p.syncCatalogSource).ToSyncerWithDelete(p.catalogSourceDeleted)), - ) - if err != nil { - return nil, err - } - p.RegisterQueueInformer(catsrcQueueInformer) + p.sources = registrygrpc.NewSourceStore(logrus.New(), stateTimeout, readyTimeout, p.syncSourceState) + p.pkgLister = pkglisters.NewPackageManifestLister(p.cache) + + // Register queue and QueueInformer + informerFactory := externalversions.NewSharedInformerFactoryWithOptions(crClient, wakeupInterval, externalversions.WithNamespace(metav1.NamespaceAll)) + catsrcInformer := informerFactory.Operators().V1alpha1().CatalogSources() + catsrcQueueInformer, err := queueinformer.NewQueueInformer( + ctx, + queueinformer.WithInformer(catsrcInformer.Informer()), + queueinformer.WithSyncer(queueinformer.LegacySyncHandler(p.syncCatalogSource).ToSyncerWithDelete(p.catalogSourceDeleted)), + ) + if err != nil { + return nil, err } + if err := p.RegisterQueueInformer(catsrcQueueInformer); err != nil { + return nil, err + } + p.catsrcLister = catsrcInformer.Lister() return p, nil } -func (p *RegistryProvider) getClient(key sourceKey) (registryClient, bool) { - p.mu.RLock() - defer p.mu.RUnlock() - - client, ok := p.clients[key] - return client, ok -} - -func (p *RegistryProvider) setClient(client registryClient, key sourceKey) { - p.mu.Lock() - defer p.mu.Unlock() - - p.clients[key] = client -} - -func (p *RegistryProvider) removeClient(key sourceKey) (registryClient, bool) { - p.mu.Lock() - defer p.mu.Unlock() - - client, ok := p.clients[key] - if !ok { - return registryClient{}, false - } - - delete(p.clients, key) - return client, true +// Run starts the provider's source connection management and catalog informers without blocking. +func (p *RegistryProvider) Run(ctx context.Context) { + p.runOnce.Do(func() { + // Both are non-blocking + p.sources.Start(ctx) + p.Operator.Run(ctx) + }) } func (p *RegistryProvider) syncCatalogSource(obj interface{}) (syncError error) { @@ -131,43 +157,176 @@ func (p *RegistryProvider) syncCatalogSource(obj interface{}) (syncError error) return } - key := sourceKey{source.GetName(), source.GetNamespace()} - client, ok := p.getClient(key) - if ok && source.Status.RegistryServiceStatus.ServiceName != "" { - logger.Info("update detected, attempting to reset grpc connection") - client.conn.ResetConnectBackoff() + address := source.Address() + logger = logger.WithField("address", address) + + key := resolver.CatalogKey{ + Namespace: source.GetNamespace(), + Name: source.GetName(), + } + if sourceMeta := p.sources.GetMeta(key); sourceMeta != nil && sourceMeta.Address == address { + // If the address hasn't changed, don't bother creating a new source + logger.Debug("catalog address unchanged, skipping source creation") + return + } + + logger.Info("connecting to source") + if _, syncError = p.sources.Add(key, address); syncError != nil { + logger.Warn("failed to create a new source") + } - ctx, cancel := context.WithTimeout(context.TODO(), defaultConnectionTimeout) - defer cancel() + return +} - changed := client.conn.WaitForStateChange(ctx, connectivity.TransientFailure) - if !changed { - logger.Debugf("grpc connection reset timeout") - syncError = fmt.Errorf("grpc connection reset timeout") - return +func (p *RegistryProvider) syncSourceState(state registrygrpc.SourceState) { + key := state.Key + logger := logrus.WithFields(logrus.Fields{ + "action": "sync source", + "source": key, + "state": state.State, + }) + logger.Debug("source state changed") + + timeout, cancel := context.WithTimeout(context.Background(), cacheTimeout) + defer cancel() + + var err error + switch state.State { + case connectivity.Ready: + var client *registryClient + client, err = p.registryClient(key) + if err == nil { + err = p.refreshCache(timeout, client) } + case connectivity.TransientFailure, connectivity.Shutdown: + err = p.gcPackages(key, nil) + default: + logger.Debug("inert source state, skipping cache update") + } - logger.Info("grpc connection reset") + if err != nil { + logger.WithError(err).Warn("failed to update cache") + } +} + +func (p *RegistryProvider) registryClient(key resolver.CatalogKey) (client *registryClient, err error) { + source := p.sources.Get(key) + if source == nil { + err = fmt.Errorf("missing source for catalog %s", key) return - } else if ok { - // Address type grpc CatalogSource, drop the connection dial in to the new address - client.conn.Close() } - logger.Info("attempting to add a new grpc connection") - conn, err := grpc.Dial(source.Address(), grpc.WithInsecure()) - if err != nil { - logger.WithField("err", err.Error()).Errorf("could not connect to registry service") - syncError = err + conn := source.Conn + if conn == nil { + err = fmt.Errorf("missing grpc connection for source %s", key) return } - p.setClient(newRegistryClient(source, conn), key) - logger.Info("new grpc connection added") + var catsrc *operatorsv1alpha1.CatalogSource + catsrc, err = p.catsrcLister.CatalogSources(key.Namespace).Get(key.Name) + if err != nil { + return + } + client = newRegistryClient(catsrc, conn) return } +func (p *RegistryProvider) refreshCache(ctx context.Context, client *registryClient) error { + key, err := client.key() + if err != nil { + return err + } + + logger := logrus.WithFields(logrus.Fields{ + "action": "refresh cache", + "source": key, + }) + + stream, err := client.ListPackages(ctx, &api.ListPackageRequest{}) + if err != nil { + logger.WithField("err", err.Error()).Warnf("error getting stream") + return nil + } + + var ( + added = map[string]struct{}{} + mu sync.Mutex + wg sync.WaitGroup + ) + for { + pkgName, err := stream.Recv() + if err == io.EOF { + break + } + if err != nil { + logger.WithField("err", err.Error()).Warnf("error getting data") + break + } + + wg.Add(1) + go func() { + defer wg.Done() + pkg, err := client.GetPackage(ctx, &api.GetPackageRequest{Name: pkgName.GetName()}) + if err != nil { + logger.WithField("err", err.Error()).Warnf("eliding package: error getting package") + return + } + + newPkg, err := newPackageManifest(ctx, logger, pkg, client) + if err != nil { + logger.WithField("err", err.Error()).Warnf("eliding package: error converting to packagemanifest") + return + } + + if err := p.cache.Add(newPkg); err != nil { + logger.WithField("err", err.Error()).Warnf("eliding package: failed to add to cache") + return + } + + mu.Lock() + defer mu.Unlock() + added[newPkg.GetName()] = struct{}{} + }() + } + + logger.Debug("caching new packages...") + wg.Wait() + logger.Debug("new packages cached") + + // Garbage collect orphaned packagemanifests from the cache + return p.gcPackages(key, added) +} + +func (p *RegistryProvider) gcPackages(key resolver.CatalogKey, keep map[string]struct{}) error { + logger := logrus.WithFields(logrus.Fields{ + "action": "gc cache", + "source": key.String(), + }) + + storedPkgKeys, err := p.cache.IndexKeys(catalogIndex, key.String()) + if err != nil { + return err + } + + var errs []error + for _, storedPkgKey := range storedPkgKeys { + _, name, _ := cache.SplitMetaNamespaceKey(storedPkgKey) + if keep != nil { + if _, ok := keep[name]; ok { + continue + } + } + + if err := p.cache.Delete(cache.ExplicitKey(storedPkgKey)); err != nil { + logger.WithField("pkg", name).WithError(err).Warn("failed to delete cache entry") + errs = append(errs, err) + } + } + + return utilerrors.NewAggregate(errs) +} + func (p *RegistryProvider) catalogSourceDeleted(obj interface{}) { catsrc, ok := obj.(metav1.Object) if !ok { @@ -186,28 +345,22 @@ func (p *RegistryProvider) catalogSourceDeleted(obj interface{}) { } } + key := resolver.CatalogKey{ + Namespace: catsrc.GetNamespace(), + Name: catsrc.GetName(), + } logger := logrus.WithFields(logrus.Fields{ - "action": "CatalogSource Deleted", - "name": catsrc.GetName(), - "namespace": catsrc.GetNamespace(), + "action": "CatalogSource Deleted", + "source": key.String(), }) - logger.Debugf("attempting to remove grpc connection") - key := sourceKey{catsrc.GetName(), catsrc.GetNamespace()} - client, removed := p.removeClient(key) - if removed { - err := client.conn.Close() - if err != nil { - logger.WithField("err", err.Error()).Error("error closing connection") - utilruntime.HandleError(fmt.Errorf("error closing connection %s", err.Error())) - return - } - logger.Debug("grpc connection removed") - return + if err := p.sources.Remove(key); err != nil { + logger.WithError(err).Warn("failed to remove source") } - logger.Debugf("no gRPC connection to remove") - + if err := p.gcPackages(key, nil); err != nil { + logger.WithError(err).Warn("failed to gc orphaned packages in cache") + } } func (p *RegistryProvider) Get(namespace, name string) (*operators.PackageManifest, error) { @@ -233,66 +386,52 @@ func (p *RegistryProvider) Get(namespace, name string) (*operators.PackageManife } func (p *RegistryProvider) List(namespace string) (*operators.PackageManifestList, error) { - logger := logrus.WithFields(logrus.Fields{ - "action": "List PackageManifests", - "namespace": namespace, - }) - - p.mu.RLock() - defer p.mu.RUnlock() - - pkgs := []operators.PackageManifest{} - for _, client := range p.clients { - if client.source.GetNamespace() == namespace || client.source.GetNamespace() == p.globalNamespace || namespace == metav1.NamespaceAll { - logger.Debugf("found CatalogSource %s", client.source.GetName()) + var pkgs []*operators.PackageManifest + if namespace == metav1.NamespaceAll { + all, err := p.pkgLister.List(labels.Everything()) + if err != nil { + return nil, err + } + pkgs = append(pkgs, all...) + } else { + nsPkgs, err := p.pkgLister.PackageManifests(namespace).List(labels.Everything()) + if err != nil { + return nil, err + } + pkgs = append(pkgs, nsPkgs...) - stream, err := client.ListPackages(context.Background(), &api.ListPackageRequest{}) + if namespace != p.globalNamespace { + globalPkgs, err := p.pkgLister.PackageManifests(p.globalNamespace).List(labels.Everything()) if err != nil { - logger.WithField("err", err.Error()).Warnf("error getting stream") - continue - } - for { - pkgName, err := stream.Recv() - if err == io.EOF { - break - } - - if err != nil { - logger.WithField("err", err.Error()).Warnf("error getting data") - break - } - pkg, err := client.GetPackage(context.Background(), &api.GetPackageRequest{Name: pkgName.GetName()}) - if err != nil { - logger.WithField("err", err.Error()).Warnf("error getting package") - continue - } - newPkg, err := toPackageManifest(logger, pkg, client) - if err != nil { - logger.WithField("err", err.Error()).Warnf("eliding package: error converting to packagemanifest") - continue - } - - // Set request namespace to stop kube clients from complaining about global namespace mismatch. - if namespace != metav1.NamespaceAll { - newPkg.SetNamespace(namespace) - } - pkgs = append(pkgs, *newPkg) + return nil, err } + + pkgs = append(pkgs, globalPkgs...) + } + } + + pkgList := &operators.PackageManifestList{} + for _, pkg := range pkgs { + out := pkg.DeepCopy() + // Set request namespace to stop k8s clients from complaining about namespace mismatch. + if namespace != metav1.NamespaceAll { + out.SetNamespace(namespace) } + pkgList.Items = append(pkgList.Items, *out) } - return &operators.PackageManifestList{Items: pkgs}, nil + return pkgList, nil } -func toPackageManifest(logger *logrus.Entry, pkg *api.Package, client registryClient) (*operators.PackageManifest, error) { +func newPackageManifest(ctx context.Context, logger *logrus.Entry, pkg *api.Package, client *registryClient) (*operators.PackageManifest, error) { pkgChannels := pkg.GetChannels() - catsrc := client.source + catsrc := client.catsrc manifest := &operators.PackageManifest{ ObjectMeta: metav1.ObjectMeta{ Name: pkg.GetName(), Namespace: catsrc.GetNamespace(), - Labels: labels.CloneAndAddLabel( - labels.CloneAndAddLabel(catsrc.GetLabels(), + Labels: utillabels.CloneAndAddLabel( + utillabels.CloneAndAddLabel(catsrc.GetLabels(), "catalog", catsrc.GetName()), "catalog-namespace", catsrc.GetNamespace()), CreationTimestamp: catsrc.GetCreationTimestamp(), }, @@ -311,7 +450,7 @@ func toPackageManifest(logger *logrus.Entry, pkg *api.Package, client registryCl defaultElided bool ) for _, pkgChannel := range pkgChannels { - bundle, err := client.GetBundleForChannel(context.Background(), &api.GetBundleInChannelRequest{PkgName: pkg.GetName(), ChannelName: pkgChannel.GetName()}) + bundle, err := client.GetBundleForChannel(ctx, &api.GetBundleInChannelRequest{PkgName: pkg.GetName(), ChannelName: pkgChannel.GetName()}) if err != nil { logger.WithError(err).WithField("channel", pkgChannel.GetName()).Warn("error getting bundle, eliding channel") defaultElided = defaultElided || pkgChannel.Name == manifest.Status.DefaultChannel diff --git a/pkg/package-server/provider/registry_test.go b/pkg/package-server/provider/registry_test.go index d6d1dead45..92eaf3d701 100644 --- a/pkg/package-server/provider/registry_test.go +++ b/pkg/package-server/provider/registry_test.go @@ -10,6 +10,7 @@ import ( "io" "net" "os" + "path/filepath" "testing" "time" @@ -55,7 +56,7 @@ func server() { logrus.Fatal(err) } - loader := sqlite.NewSQLLoaderForDirectory(load, "manifests") + loader := sqlite.NewSQLLoaderForDirectory(load, filepath.Join("testdata", "manifests")) if err := loader.Populate(); err != nil { logrus.Fatal(err) } @@ -74,7 +75,7 @@ func server() { } } -func NewFakeRegistryProvider(ctx context.Context, clientObjs []runtime.Object, k8sObjs []runtime.Object, watchedNamespaces []string, globalNamespace string) (*RegistryProvider, error) { +func NewFakeRegistryProvider(ctx context.Context, clientObjs []runtime.Object, k8sObjs []runtime.Object, globalNamespace string) (*RegistryProvider, error) { clientFake := fake.NewSimpleClientset(clientObjs...) k8sClientFake := k8sfake.NewSimpleClientset(k8sObjs...) opClientFake := operatorclient.NewClient(k8sClientFake, nil, nil) @@ -86,7 +87,7 @@ func NewFakeRegistryProvider(ctx context.Context, clientObjs []runtime.Object, k resyncInterval := 5 * time.Minute - return NewRegistryProvider(ctx, clientFake, op, resyncInterval, watchedNamespaces, globalNamespace) + return NewRegistryProvider(ctx, clientFake, op, resyncInterval, globalNamespace) } func catalogSource(name, namespace string) *operatorsv1alpha1.CatalogSource { @@ -349,12 +350,12 @@ func TestToPackageManifest(t *testing.T) { clientFake := &fakes.FakeRegistryClient{} clientFake.GetBundleForChannelReturnsOnCall(0, test.bundle, nil) - client := registryClient{ + client := ®istryClient{ RegistryClient: clientFake, - source: test.catalogSource, + catsrc: test.catalogSource, } - packageManifest, err := toPackageManifest(logrus.NewEntry(logrus.New()), test.apiPkg, client) + packageManifest, err := newPackageManifest(context.Background(), logrus.NewEntry(logrus.New()), test.apiPkg, client) if test.expectedErr != "" { require.Error(t, err) require.Equal(t, test.expectedErr, err.Error()) @@ -539,14 +540,12 @@ func TestRegistryProviderGet(t *testing.T) { t.Run(test.name, func(t *testing.T) { ctx, cancel := context.WithCancel(context.TODO()) defer cancel() - provider, err := NewFakeRegistryProvider(ctx, nil, nil, test.namespaces, test.globalNS) + provider, err := NewFakeRegistryProvider(ctx, nil, nil, test.globalNS) require.NoError(t, err) for _, cs := range test.catalogSources { - catsrc := (cs).(*operatorsv1alpha1.CatalogSource) - conn, err := grpc.Dial(address+catsrc.Status.RegistryServiceStatus.Port, grpc.WithInsecure()) - require.NoError(t, err, "could not set up test grpc connection") - provider.clients[sourceKey{catsrc.GetName(), catsrc.GetNamespace()}] = newRegistryClient(catsrc, conn) + catsrc := cs.(*operatorsv1alpha1.CatalogSource) + require.NoError(t, provider.refreshCache(ctx, newTestRegistryClient(t, catsrc))) } packageManifest, err := provider.Get(test.request.packageNamespace, test.request.packageName) @@ -565,26 +564,23 @@ func TestRegistryProviderGet(t *testing.T) { func TestRegistryProviderList(t *testing.T) { tests := []struct { name string - namespaces []string globalNS string - registryClients []registryClient + registryClients []*registryClient requestNamespace string expectedErr string expected *operators.PackageManifestList }{ { name: "NoPackages", - namespaces: []string{"ns"}, globalNS: "ns", requestNamespace: "wisconsin", expectedErr: "", expected: &operators.PackageManifestList{Items: []operators.PackageManifest{}}, }, { - name: "PackagesFound", - namespaces: []string{"ns"}, - globalNS: "ns", - registryClients: []registryClient{ + name: "PackagesFound", + globalNS: "ns", + registryClients: []*registryClient{ newTestRegistryClient(t, withRegistryServiceStatus(catalogSource("cool-operators", "ns"), "grpc", "cool-operators", "ns", port, metav1.NewTime(time.Now()))), }, requestNamespace: "ns", @@ -657,10 +653,9 @@ func TestRegistryProviderList(t *testing.T) { }}, }, { - name: "TwoCatalogs/OneBadConnection/PackagesFound", - namespaces: []string{"ns"}, - globalNS: "ns", - registryClients: []registryClient{ + name: "TwoCatalogs/OneBadConnection/PackagesFound", + globalNS: "ns", + registryClients: []*registryClient{ newTestRegistryClient(t, withRegistryServiceStatus(catalogSource("cool-operators", "ns"), "grpc", "cool-operators", "ns", port, metav1.NewTime(time.Now()))), newTestRegistryClient(t, withRegistryServiceStatus(catalogSource("not-so-cool-operators", "ns"), "grpc", "not-so-cool-operators", "ns", "50052", metav1.NewTime(time.Now()))), }, @@ -734,12 +729,11 @@ func TestRegistryProviderList(t *testing.T) { }}, }, { - name: "OneCatalog/ManyPackages/OneMissingBundle/Elided", - namespaces: []string{"ns"}, - globalNS: "ns", - registryClients: []registryClient{ - func() registryClient { - source := catalogSource("cool-operators", "ns") + name: "OneCatalog/ManyPackages/OneMissingBundle/Elided", + globalNS: "ns", + registryClients: []*registryClient{ + func() *registryClient { + catsrc := catalogSource("cool-operators", "ns") listFake := &fakes.FakeRegistry_ListPackagesClient{} listFake.RecvReturnsOnCall(0, &api.PackageName{Name: "no-bundle"}, nil) listFake.RecvReturnsOnCall(1, &api.PackageName{Name: "has-bundle"}, nil) @@ -781,7 +775,7 @@ func TestRegistryProviderList(t *testing.T) { }, }, nil) - return registryClient{clientFake, source, nil} + return ®istryClient{clientFake, catsrc, nil} }(), }, requestNamespace: "ns", @@ -827,11 +821,11 @@ func TestRegistryProviderList(t *testing.T) { t.Run(test.name, func(t *testing.T) { ctx, cancel := context.WithCancel(context.TODO()) defer cancel() - provider, err := NewFakeRegistryProvider(ctx, nil, nil, test.namespaces, test.globalNS) + provider, err := NewFakeRegistryProvider(ctx, nil, nil, test.globalNS) require.NoError(t, err) for _, c := range test.registryClients { - provider.clients[sourceKey{c.source.GetName(), c.source.GetNamespace()}] = c + require.NoError(t, provider.refreshCache(ctx, c)) } packageManifestList, err := provider.List(test.requestNamespace) @@ -848,8 +842,8 @@ func TestRegistryProviderList(t *testing.T) { } } -func newTestRegistryClient(t *testing.T, source *operatorsv1alpha1.CatalogSource) registryClient { - conn, err := grpc.Dial(address+source.Status.RegistryServiceStatus.Port, grpc.WithInsecure()) +func newTestRegistryClient(t *testing.T, catsrc *operatorsv1alpha1.CatalogSource) *registryClient { + conn, err := grpc.Dial(address+catsrc.Status.RegistryServiceStatus.Port, grpc.WithInsecure()) require.NoError(t, err, "could not set up test grpc connection") - return newRegistryClient(source, conn) + return newRegistryClient(catsrc, conn) } diff --git a/pkg/package-server/provider/manifests/etcd/0.6.1/etcdcluster.crd.yaml b/pkg/package-server/provider/testdata/manifests/etcd/0.6.1/etcdcluster.crd.yaml similarity index 100% rename from pkg/package-server/provider/manifests/etcd/0.6.1/etcdcluster.crd.yaml rename to pkg/package-server/provider/testdata/manifests/etcd/0.6.1/etcdcluster.crd.yaml diff --git a/pkg/package-server/provider/manifests/etcd/0.6.1/etcdoperator.clusterserviceversion.yaml b/pkg/package-server/provider/testdata/manifests/etcd/0.6.1/etcdoperator.clusterserviceversion.yaml similarity index 100% rename from pkg/package-server/provider/manifests/etcd/0.6.1/etcdoperator.clusterserviceversion.yaml rename to pkg/package-server/provider/testdata/manifests/etcd/0.6.1/etcdoperator.clusterserviceversion.yaml diff --git a/pkg/package-server/provider/manifests/etcd/0.9.0/etcdbackup.crd.yaml b/pkg/package-server/provider/testdata/manifests/etcd/0.9.0/etcdbackup.crd.yaml similarity index 100% rename from pkg/package-server/provider/manifests/etcd/0.9.0/etcdbackup.crd.yaml rename to pkg/package-server/provider/testdata/manifests/etcd/0.9.0/etcdbackup.crd.yaml diff --git a/pkg/package-server/provider/manifests/etcd/0.9.0/etcdcluster.crd.yaml b/pkg/package-server/provider/testdata/manifests/etcd/0.9.0/etcdcluster.crd.yaml similarity index 100% rename from pkg/package-server/provider/manifests/etcd/0.9.0/etcdcluster.crd.yaml rename to pkg/package-server/provider/testdata/manifests/etcd/0.9.0/etcdcluster.crd.yaml diff --git a/pkg/package-server/provider/manifests/etcd/0.9.0/etcdoperator.v0.9.0.clusterserviceversion.yaml b/pkg/package-server/provider/testdata/manifests/etcd/0.9.0/etcdoperator.v0.9.0.clusterserviceversion.yaml similarity index 100% rename from pkg/package-server/provider/manifests/etcd/0.9.0/etcdoperator.v0.9.0.clusterserviceversion.yaml rename to pkg/package-server/provider/testdata/manifests/etcd/0.9.0/etcdoperator.v0.9.0.clusterserviceversion.yaml diff --git a/pkg/package-server/provider/manifests/etcd/0.9.0/etcdrestore.crd.yaml b/pkg/package-server/provider/testdata/manifests/etcd/0.9.0/etcdrestore.crd.yaml similarity index 100% rename from pkg/package-server/provider/manifests/etcd/0.9.0/etcdrestore.crd.yaml rename to pkg/package-server/provider/testdata/manifests/etcd/0.9.0/etcdrestore.crd.yaml diff --git a/pkg/package-server/provider/manifests/etcd/0.9.2/etcdbackup.crd.yaml b/pkg/package-server/provider/testdata/manifests/etcd/0.9.2/etcdbackup.crd.yaml similarity index 100% rename from pkg/package-server/provider/manifests/etcd/0.9.2/etcdbackup.crd.yaml rename to pkg/package-server/provider/testdata/manifests/etcd/0.9.2/etcdbackup.crd.yaml diff --git a/pkg/package-server/provider/manifests/etcd/0.9.2/etcdcluster.crd.yaml b/pkg/package-server/provider/testdata/manifests/etcd/0.9.2/etcdcluster.crd.yaml similarity index 100% rename from pkg/package-server/provider/manifests/etcd/0.9.2/etcdcluster.crd.yaml rename to pkg/package-server/provider/testdata/manifests/etcd/0.9.2/etcdcluster.crd.yaml diff --git a/pkg/package-server/provider/manifests/etcd/0.9.2/etcdoperator.v0.9.2.clusterserviceversion.yaml b/pkg/package-server/provider/testdata/manifests/etcd/0.9.2/etcdoperator.v0.9.2.clusterserviceversion.yaml similarity index 100% rename from pkg/package-server/provider/manifests/etcd/0.9.2/etcdoperator.v0.9.2.clusterserviceversion.yaml rename to pkg/package-server/provider/testdata/manifests/etcd/0.9.2/etcdoperator.v0.9.2.clusterserviceversion.yaml diff --git a/pkg/package-server/provider/manifests/etcd/0.9.2/etcdrestore.crd.yaml b/pkg/package-server/provider/testdata/manifests/etcd/0.9.2/etcdrestore.crd.yaml similarity index 100% rename from pkg/package-server/provider/manifests/etcd/0.9.2/etcdrestore.crd.yaml rename to pkg/package-server/provider/testdata/manifests/etcd/0.9.2/etcdrestore.crd.yaml diff --git a/pkg/package-server/provider/manifests/etcd/etcd.package.yaml b/pkg/package-server/provider/testdata/manifests/etcd/etcd.package.yaml similarity index 100% rename from pkg/package-server/provider/manifests/etcd/etcd.package.yaml rename to pkg/package-server/provider/testdata/manifests/etcd/etcd.package.yaml diff --git a/pkg/package-server/provider/manifests/prometheus/0.14.0/alertmanager.crd.yaml b/pkg/package-server/provider/testdata/manifests/prometheus/0.14.0/alertmanager.crd.yaml similarity index 100% rename from pkg/package-server/provider/manifests/prometheus/0.14.0/alertmanager.crd.yaml rename to pkg/package-server/provider/testdata/manifests/prometheus/0.14.0/alertmanager.crd.yaml diff --git a/pkg/package-server/provider/manifests/prometheus/0.14.0/prometheus.crd.yaml b/pkg/package-server/provider/testdata/manifests/prometheus/0.14.0/prometheus.crd.yaml similarity index 100% rename from pkg/package-server/provider/manifests/prometheus/0.14.0/prometheus.crd.yaml rename to pkg/package-server/provider/testdata/manifests/prometheus/0.14.0/prometheus.crd.yaml diff --git a/pkg/package-server/provider/manifests/prometheus/0.14.0/prometheusoperator.0.14.0.clusterserviceversion.yaml b/pkg/package-server/provider/testdata/manifests/prometheus/0.14.0/prometheusoperator.0.14.0.clusterserviceversion.yaml similarity index 100% rename from pkg/package-server/provider/manifests/prometheus/0.14.0/prometheusoperator.0.14.0.clusterserviceversion.yaml rename to pkg/package-server/provider/testdata/manifests/prometheus/0.14.0/prometheusoperator.0.14.0.clusterserviceversion.yaml diff --git a/pkg/package-server/provider/manifests/prometheus/0.14.0/prometheusrule.crd.yaml b/pkg/package-server/provider/testdata/manifests/prometheus/0.14.0/prometheusrule.crd.yaml similarity index 100% rename from pkg/package-server/provider/manifests/prometheus/0.14.0/prometheusrule.crd.yaml rename to pkg/package-server/provider/testdata/manifests/prometheus/0.14.0/prometheusrule.crd.yaml diff --git a/pkg/package-server/provider/manifests/prometheus/0.14.0/servicemonitor.crd.yaml b/pkg/package-server/provider/testdata/manifests/prometheus/0.14.0/servicemonitor.crd.yaml similarity index 100% rename from pkg/package-server/provider/manifests/prometheus/0.14.0/servicemonitor.crd.yaml rename to pkg/package-server/provider/testdata/manifests/prometheus/0.14.0/servicemonitor.crd.yaml diff --git a/pkg/package-server/provider/manifests/prometheus/0.15.0/alertmanager.crd.yaml b/pkg/package-server/provider/testdata/manifests/prometheus/0.15.0/alertmanager.crd.yaml similarity index 100% rename from pkg/package-server/provider/manifests/prometheus/0.15.0/alertmanager.crd.yaml rename to pkg/package-server/provider/testdata/manifests/prometheus/0.15.0/alertmanager.crd.yaml diff --git a/pkg/package-server/provider/manifests/prometheus/0.15.0/prometheus.crd.yaml b/pkg/package-server/provider/testdata/manifests/prometheus/0.15.0/prometheus.crd.yaml similarity index 100% rename from pkg/package-server/provider/manifests/prometheus/0.15.0/prometheus.crd.yaml rename to pkg/package-server/provider/testdata/manifests/prometheus/0.15.0/prometheus.crd.yaml diff --git a/pkg/package-server/provider/manifests/prometheus/0.15.0/prometheusoperator.0.15.0.clusterserviceversion.yaml b/pkg/package-server/provider/testdata/manifests/prometheus/0.15.0/prometheusoperator.0.15.0.clusterserviceversion.yaml similarity index 100% rename from pkg/package-server/provider/manifests/prometheus/0.15.0/prometheusoperator.0.15.0.clusterserviceversion.yaml rename to pkg/package-server/provider/testdata/manifests/prometheus/0.15.0/prometheusoperator.0.15.0.clusterserviceversion.yaml diff --git a/pkg/package-server/provider/manifests/prometheus/0.15.0/prometheusrule.crd.yaml b/pkg/package-server/provider/testdata/manifests/prometheus/0.15.0/prometheusrule.crd.yaml similarity index 100% rename from pkg/package-server/provider/manifests/prometheus/0.15.0/prometheusrule.crd.yaml rename to pkg/package-server/provider/testdata/manifests/prometheus/0.15.0/prometheusrule.crd.yaml diff --git a/pkg/package-server/provider/manifests/prometheus/0.15.0/servicemonitor.crd.yaml b/pkg/package-server/provider/testdata/manifests/prometheus/0.15.0/servicemonitor.crd.yaml similarity index 100% rename from pkg/package-server/provider/manifests/prometheus/0.15.0/servicemonitor.crd.yaml rename to pkg/package-server/provider/testdata/manifests/prometheus/0.15.0/servicemonitor.crd.yaml diff --git a/pkg/package-server/provider/manifests/prometheus/0.22.2/alertmanager.crd.yaml b/pkg/package-server/provider/testdata/manifests/prometheus/0.22.2/alertmanager.crd.yaml similarity index 100% rename from pkg/package-server/provider/manifests/prometheus/0.22.2/alertmanager.crd.yaml rename to pkg/package-server/provider/testdata/manifests/prometheus/0.22.2/alertmanager.crd.yaml diff --git a/pkg/package-server/provider/manifests/prometheus/0.22.2/prometheus.crd.yaml b/pkg/package-server/provider/testdata/manifests/prometheus/0.22.2/prometheus.crd.yaml similarity index 100% rename from pkg/package-server/provider/manifests/prometheus/0.22.2/prometheus.crd.yaml rename to pkg/package-server/provider/testdata/manifests/prometheus/0.22.2/prometheus.crd.yaml diff --git a/pkg/package-server/provider/manifests/prometheus/0.22.2/prometheusoperator.0.22.2.clusterserviceversion.yaml b/pkg/package-server/provider/testdata/manifests/prometheus/0.22.2/prometheusoperator.0.22.2.clusterserviceversion.yaml similarity index 100% rename from pkg/package-server/provider/manifests/prometheus/0.22.2/prometheusoperator.0.22.2.clusterserviceversion.yaml rename to pkg/package-server/provider/testdata/manifests/prometheus/0.22.2/prometheusoperator.0.22.2.clusterserviceversion.yaml diff --git a/pkg/package-server/provider/manifests/prometheus/0.22.2/prometheusrule.crd.yaml b/pkg/package-server/provider/testdata/manifests/prometheus/0.22.2/prometheusrule.crd.yaml similarity index 100% rename from pkg/package-server/provider/manifests/prometheus/0.22.2/prometheusrule.crd.yaml rename to pkg/package-server/provider/testdata/manifests/prometheus/0.22.2/prometheusrule.crd.yaml diff --git a/pkg/package-server/provider/manifests/prometheus/0.22.2/servicemonitor.crd.yaml b/pkg/package-server/provider/testdata/manifests/prometheus/0.22.2/servicemonitor.crd.yaml similarity index 100% rename from pkg/package-server/provider/manifests/prometheus/0.22.2/servicemonitor.crd.yaml rename to pkg/package-server/provider/testdata/manifests/prometheus/0.22.2/servicemonitor.crd.yaml diff --git a/pkg/package-server/provider/manifests/prometheus/prometheus.package.yaml b/pkg/package-server/provider/testdata/manifests/prometheus/prometheus.package.yaml similarity index 100% rename from pkg/package-server/provider/manifests/prometheus/prometheus.package.yaml rename to pkg/package-server/provider/testdata/manifests/prometheus/prometheus.package.yaml diff --git a/pkg/package-server/server/server.go b/pkg/package-server/server/server.go index 46e395ffd4..73e5493abc 100644 --- a/pkg/package-server/server/server.go +++ b/pkg/package-server/server/server.go @@ -9,7 +9,6 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" genericserver "k8s.io/apiserver/pkg/server" genericoptions "k8s.io/apiserver/pkg/server/options" "k8s.io/client-go/informers" @@ -39,10 +38,8 @@ func NewCommandStartPackageServer(ctx context.Context, defaults *PackageServerOp } flags := cmd.Flags() - - // flags.BoolVar(&defaults.InsecureKubeletTLS, "kubelet-insecure-tls", defaults.InsecureKubeletTLS, "Do not verify CA of serving certificates presented by Kubelets. For testing purposes only.") flags.DurationVar(&defaults.WakeupInterval, "interval", defaults.WakeupInterval, "interval at which to re-sync CatalogSources") - flags.StringSliceVar(&defaults.WatchedNamespaces, "watched-namespaces", defaults.WatchedNamespaces, "list of namespaces the package-server will watch watch for CatalogSources") + flags.StringVar(&defaults.GlobalNamespace, "global-namespace", defaults.GlobalNamespace, "Name of the namespace where the global CatalogSources are located") flags.StringVar(&defaults.Kubeconfig, "kubeconfig", defaults.Kubeconfig, "path to the kubeconfig used to connect to the Kubernetes API server and the Kubelets (defaults to in-cluster config)") flags.BoolVar(&defaults.Debug, "debug", defaults.Debug, "use debug log level") @@ -61,9 +58,8 @@ type PackageServerOptions struct { Authorization *genericoptions.DelegatingAuthorizationOptions Features *genericoptions.FeatureOptions - GlobalNamespace string - WatchedNamespaces []string - WakeupInterval time.Duration + GlobalNamespace string + WakeupInterval time.Duration Kubeconfig string RegistryAddr string @@ -87,8 +83,7 @@ func NewPackageServerOptions(out, errOut io.Writer) *PackageServerOptions { Authorization: genericoptions.NewDelegatingAuthorizationOptions(), Features: genericoptions.NewFeatureOptions(), - WatchedNamespaces: []string{v1.NamespaceAll}, - WakeupInterval: 5 * time.Minute, + WakeupInterval: 5 * time.Minute, DisableAuthForTesting: false, Debug: false, @@ -100,10 +95,7 @@ func NewPackageServerOptions(out, errOut io.Writer) *PackageServerOptions { return o } -func (o *PackageServerOptions) Complete() error { - return nil -} - +// Config returns config for the PackageServerOptions. func (o *PackageServerOptions) Config() (*apiserver.Config, error) { if err := o.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{net.ParseIP("127.0.0.1")}); err != nil { return nil, fmt.Errorf("error creating self-signed certificates: %v", err) @@ -129,6 +121,7 @@ func (o *PackageServerOptions) Config() (*apiserver.Config, error) { }, nil } +// Run starts a new packageserver for the PackageServerOptions. func (o *PackageServerOptions) Run(ctx context.Context) error { if o.Debug { log.SetLevel(log.DebugLevel) @@ -170,7 +163,7 @@ func (o *PackageServerOptions) Run(ctx context.Context) error { return err } - sourceProvider, err := provider.NewRegistryProvider(ctx, crClient, queueOperator, o.WakeupInterval, o.WatchedNamespaces, o.GlobalNamespace) + sourceProvider, err := provider.NewRegistryProvider(ctx, crClient, queueOperator, o.WakeupInterval, o.GlobalNamespace) if err != nil { return err } diff --git a/test/e2e/setup_test.go b/test/e2e/setup_test.go index 6510893fa7..0bd07009ce 100644 --- a/test/e2e/setup_test.go +++ b/test/e2e/setup_test.go @@ -39,9 +39,6 @@ var ( ) func TestMain(m *testing.M) { - if err := flag.Set("logtostderr", "true"); err != nil { - panic(err) - } flag.Parse() testNamespace = *namespace diff --git a/vendor/github.com/golang/glog/LICENSE b/vendor/github.com/golang/glog/LICENSE deleted file mode 100644 index 37ec93a14f..0000000000 --- a/vendor/github.com/golang/glog/LICENSE +++ /dev/null @@ -1,191 +0,0 @@ -Apache License -Version 2.0, January 2004 -http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - -"License" shall mean the terms and conditions for use, reproduction, and -distribution as defined by Sections 1 through 9 of this document. - -"Licensor" shall mean the copyright owner or entity authorized by the copyright -owner that is granting the License. - -"Legal Entity" shall mean the union of the acting entity and all other entities -that control, are controlled by, or are under common control with that entity. -For the purposes of this definition, "control" means (i) the power, direct or -indirect, to cause the direction or management of such entity, whether by -contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the -outstanding shares, or (iii) beneficial ownership of such entity. - -"You" (or "Your") shall mean an individual or Legal Entity exercising -permissions granted by this License. - -"Source" form shall mean the preferred form for making modifications, including -but not limited to software source code, documentation source, and configuration -files. - -"Object" form shall mean any form resulting from mechanical transformation or -translation of a Source form, including but not limited to compiled object code, -generated documentation, and conversions to other media types. - -"Work" shall mean the work of authorship, whether in Source or Object form, made -available under the License, as indicated by a copyright notice that is included -in or attached to the work (an example is provided in the Appendix below). - -"Derivative Works" shall mean any work, whether in Source or Object form, that -is based on (or derived from) the Work and for which the editorial revisions, -annotations, elaborations, or other modifications represent, as a whole, an -original work of authorship. For the purposes of this License, Derivative Works -shall not include works that remain separable from, or merely link (or bind by -name) to the interfaces of, the Work and Derivative Works thereof. - -"Contribution" shall mean any work of authorship, including the original version -of the Work and any modifications or additions to that Work or Derivative Works -thereof, that is intentionally submitted to Licensor for inclusion in the Work -by the copyright owner or by an individual or Legal Entity authorized to submit -on behalf of the copyright owner. For the purposes of this definition, -"submitted" means any form of electronic, verbal, or written communication sent -to the Licensor or its representatives, including but not limited to -communication on electronic mailing lists, source code control systems, and -issue tracking systems that are managed by, or on behalf of, the Licensor for -the purpose of discussing and improving the Work, but excluding communication -that is conspicuously marked or otherwise designated in writing by the copyright -owner as "Not a Contribution." - -"Contributor" shall mean Licensor and any individual or Legal Entity on behalf -of whom a Contribution has been received by Licensor and subsequently -incorporated within the Work. - -2. Grant of Copyright License. - -Subject to the terms and conditions of this License, each Contributor hereby -grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, -irrevocable copyright license to reproduce, prepare Derivative Works of, -publicly display, publicly perform, sublicense, and distribute the Work and such -Derivative Works in Source or Object form. - -3. Grant of Patent License. - -Subject to the terms and conditions of this License, each Contributor hereby -grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, -irrevocable (except as stated in this section) patent license to make, have -made, use, offer to sell, sell, import, and otherwise transfer the Work, where -such license applies only to those patent claims licensable by such Contributor -that are necessarily infringed by their Contribution(s) alone or by combination -of their Contribution(s) with the Work to which such Contribution(s) was -submitted. If You institute patent litigation against any entity (including a -cross-claim or counterclaim in a lawsuit) alleging that the Work or a -Contribution incorporated within the Work constitutes direct or contributory -patent infringement, then any patent licenses granted to You under this License -for that Work shall terminate as of the date such litigation is filed. - -4. Redistribution. - -You may reproduce and distribute copies of the Work or Derivative Works thereof -in any medium, with or without modifications, and in Source or Object form, -provided that You meet the following conditions: - -You must give any other recipients of the Work or Derivative Works a copy of -this License; and -You must cause any modified files to carry prominent notices stating that You -changed the files; and -You must retain, in the Source form of any Derivative Works that You distribute, -all copyright, patent, trademark, and attribution notices from the Source form -of the Work, excluding those notices that do not pertain to any part of the -Derivative Works; and -If the Work includes a "NOTICE" text file as part of its distribution, then any -Derivative Works that You distribute must include a readable copy of the -attribution notices contained within such NOTICE file, excluding those notices -that do not pertain to any part of the Derivative Works, in at least one of the -following places: within a NOTICE text file distributed as part of the -Derivative Works; within the Source form or documentation, if provided along -with the Derivative Works; or, within a display generated by the Derivative -Works, if and wherever such third-party notices normally appear. The contents of -the NOTICE file are for informational purposes only and do not modify the -License. You may add Your own attribution notices within Derivative Works that -You distribute, alongside or as an addendum to the NOTICE text from the Work, -provided that such additional attribution notices cannot be construed as -modifying the License. -You may add Your own copyright statement to Your modifications and may provide -additional or different license terms and conditions for use, reproduction, or -distribution of Your modifications, or for any such Derivative Works as a whole, -provided Your use, reproduction, and distribution of the Work otherwise complies -with the conditions stated in this License. - -5. Submission of Contributions. - -Unless You explicitly state otherwise, any Contribution intentionally submitted -for inclusion in the Work by You to the Licensor shall be under the terms and -conditions of this License, without any additional terms or conditions. -Notwithstanding the above, nothing herein shall supersede or modify the terms of -any separate license agreement you may have executed with Licensor regarding -such Contributions. - -6. Trademarks. - -This License does not grant permission to use the trade names, trademarks, -service marks, or product names of the Licensor, except as required for -reasonable and customary use in describing the origin of the Work and -reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. - -Unless required by applicable law or agreed to in writing, Licensor provides the -Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, -including, without limitation, any warranties or conditions of TITLE, -NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are -solely responsible for determining the appropriateness of using or -redistributing the Work and assume any risks associated with Your exercise of -permissions under this License. - -8. Limitation of Liability. - -In no event and under no legal theory, whether in tort (including negligence), -contract, or otherwise, unless required by applicable law (such as deliberate -and grossly negligent acts) or agreed to in writing, shall any Contributor be -liable to You for damages, including any direct, indirect, special, incidental, -or consequential damages of any character arising as a result of this License or -out of the use or inability to use the Work (including but not limited to -damages for loss of goodwill, work stoppage, computer failure or malfunction, or -any and all other commercial damages or losses), even if such Contributor has -been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. - -While redistributing the Work or Derivative Works thereof, You may choose to -offer, and charge a fee for, acceptance of support, warranty, indemnity, or -other liability obligations and/or rights consistent with this License. However, -in accepting such obligations, You may act only on Your own behalf and on Your -sole responsibility, not on behalf of any other Contributor, and only if You -agree to indemnify, defend, and hold each Contributor harmless for any liability -incurred by, or claims asserted against, such Contributor by reason of your -accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS - -APPENDIX: How to apply the Apache License to your work - -To apply the Apache License to your work, attach the following boilerplate -notice, with the fields enclosed by brackets "[]" replaced with your own -identifying information. (Don't include the brackets!) The text should be -enclosed in the appropriate comment syntax for the file format. We also -recommend that a file or class name and description of purpose be included on -the same "printed page" as the copyright notice for easier identification within -third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/vendor/github.com/golang/glog/README b/vendor/github.com/golang/glog/README deleted file mode 100644 index 387b4eb689..0000000000 --- a/vendor/github.com/golang/glog/README +++ /dev/null @@ -1,44 +0,0 @@ -glog -==== - -Leveled execution logs for Go. - -This is an efficient pure Go implementation of leveled logs in the -manner of the open source C++ package - https://github.com/google/glog - -By binding methods to booleans it is possible to use the log package -without paying the expense of evaluating the arguments to the log. -Through the -vmodule flag, the package also provides fine-grained -control over logging at the file level. - -The comment from glog.go introduces the ideas: - - Package glog implements logging analogous to the Google-internal - C++ INFO/ERROR/V setup. It provides functions Info, Warning, - Error, Fatal, plus formatting variants such as Infof. It - also provides V-style logging controlled by the -v and - -vmodule=file=2 flags. - - Basic examples: - - glog.Info("Prepare to repel boarders") - - glog.Fatalf("Initialization failed: %s", err) - - See the documentation for the V function for an explanation - of these examples: - - if glog.V(2) { - glog.Info("Starting transaction...") - } - - glog.V(2).Infoln("Processed", nItems, "elements") - - -The repository contains an open source version of the log package -used inside Google. The master copy of the source lives inside -Google, not here. The code in this repo is for export only and is not itself -under development. Feature requests will be ignored. - -Send bug reports to golang-nuts@googlegroups.com. diff --git a/vendor/github.com/golang/glog/glog.go b/vendor/github.com/golang/glog/glog.go deleted file mode 100644 index 54bd7afdca..0000000000 --- a/vendor/github.com/golang/glog/glog.go +++ /dev/null @@ -1,1180 +0,0 @@ -// Go support for leveled logs, analogous to https://code.google.com/p/google-glog/ -// -// Copyright 2013 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package glog implements logging analogous to the Google-internal C++ INFO/ERROR/V setup. -// It provides functions Info, Warning, Error, Fatal, plus formatting variants such as -// Infof. It also provides V-style logging controlled by the -v and -vmodule=file=2 flags. -// -// Basic examples: -// -// glog.Info("Prepare to repel boarders") -// -// glog.Fatalf("Initialization failed: %s", err) -// -// See the documentation for the V function for an explanation of these examples: -// -// if glog.V(2) { -// glog.Info("Starting transaction...") -// } -// -// glog.V(2).Infoln("Processed", nItems, "elements") -// -// Log output is buffered and written periodically using Flush. Programs -// should call Flush before exiting to guarantee all log output is written. -// -// By default, all log statements write to files in a temporary directory. -// This package provides several flags that modify this behavior. -// As a result, flag.Parse must be called before any logging is done. -// -// -logtostderr=false -// Logs are written to standard error instead of to files. -// -alsologtostderr=false -// Logs are written to standard error as well as to files. -// -stderrthreshold=ERROR -// Log events at or above this severity are logged to standard -// error as well as to files. -// -log_dir="" -// Log files will be written to this directory instead of the -// default temporary directory. -// -// Other flags provide aids to debugging. -// -// -log_backtrace_at="" -// When set to a file and line number holding a logging statement, -// such as -// -log_backtrace_at=gopherflakes.go:234 -// a stack trace will be written to the Info log whenever execution -// hits that statement. (Unlike with -vmodule, the ".go" must be -// present.) -// -v=0 -// Enable V-leveled logging at the specified level. -// -vmodule="" -// The syntax of the argument is a comma-separated list of pattern=N, -// where pattern is a literal file name (minus the ".go" suffix) or -// "glob" pattern and N is a V level. For instance, -// -vmodule=gopher*=3 -// sets the V level to 3 in all Go files whose names begin "gopher". -// -package glog - -import ( - "bufio" - "bytes" - "errors" - "flag" - "fmt" - "io" - stdLog "log" - "os" - "path/filepath" - "runtime" - "strconv" - "strings" - "sync" - "sync/atomic" - "time" -) - -// severity identifies the sort of log: info, warning etc. It also implements -// the flag.Value interface. The -stderrthreshold flag is of type severity and -// should be modified only through the flag.Value interface. The values match -// the corresponding constants in C++. -type severity int32 // sync/atomic int32 - -// These constants identify the log levels in order of increasing severity. -// A message written to a high-severity log file is also written to each -// lower-severity log file. -const ( - infoLog severity = iota - warningLog - errorLog - fatalLog - numSeverity = 4 -) - -const severityChar = "IWEF" - -var severityName = []string{ - infoLog: "INFO", - warningLog: "WARNING", - errorLog: "ERROR", - fatalLog: "FATAL", -} - -// get returns the value of the severity. -func (s *severity) get() severity { - return severity(atomic.LoadInt32((*int32)(s))) -} - -// set sets the value of the severity. -func (s *severity) set(val severity) { - atomic.StoreInt32((*int32)(s), int32(val)) -} - -// String is part of the flag.Value interface. -func (s *severity) String() string { - return strconv.FormatInt(int64(*s), 10) -} - -// Get is part of the flag.Value interface. -func (s *severity) Get() interface{} { - return *s -} - -// Set is part of the flag.Value interface. -func (s *severity) Set(value string) error { - var threshold severity - // Is it a known name? - if v, ok := severityByName(value); ok { - threshold = v - } else { - v, err := strconv.Atoi(value) - if err != nil { - return err - } - threshold = severity(v) - } - logging.stderrThreshold.set(threshold) - return nil -} - -func severityByName(s string) (severity, bool) { - s = strings.ToUpper(s) - for i, name := range severityName { - if name == s { - return severity(i), true - } - } - return 0, false -} - -// OutputStats tracks the number of output lines and bytes written. -type OutputStats struct { - lines int64 - bytes int64 -} - -// Lines returns the number of lines written. -func (s *OutputStats) Lines() int64 { - return atomic.LoadInt64(&s.lines) -} - -// Bytes returns the number of bytes written. -func (s *OutputStats) Bytes() int64 { - return atomic.LoadInt64(&s.bytes) -} - -// Stats tracks the number of lines of output and number of bytes -// per severity level. Values must be read with atomic.LoadInt64. -var Stats struct { - Info, Warning, Error OutputStats -} - -var severityStats = [numSeverity]*OutputStats{ - infoLog: &Stats.Info, - warningLog: &Stats.Warning, - errorLog: &Stats.Error, -} - -// Level is exported because it appears in the arguments to V and is -// the type of the v flag, which can be set programmatically. -// It's a distinct type because we want to discriminate it from logType. -// Variables of type level are only changed under logging.mu. -// The -v flag is read only with atomic ops, so the state of the logging -// module is consistent. - -// Level is treated as a sync/atomic int32. - -// Level specifies a level of verbosity for V logs. *Level implements -// flag.Value; the -v flag is of type Level and should be modified -// only through the flag.Value interface. -type Level int32 - -// get returns the value of the Level. -func (l *Level) get() Level { - return Level(atomic.LoadInt32((*int32)(l))) -} - -// set sets the value of the Level. -func (l *Level) set(val Level) { - atomic.StoreInt32((*int32)(l), int32(val)) -} - -// String is part of the flag.Value interface. -func (l *Level) String() string { - return strconv.FormatInt(int64(*l), 10) -} - -// Get is part of the flag.Value interface. -func (l *Level) Get() interface{} { - return *l -} - -// Set is part of the flag.Value interface. -func (l *Level) Set(value string) error { - v, err := strconv.Atoi(value) - if err != nil { - return err - } - logging.mu.Lock() - defer logging.mu.Unlock() - logging.setVState(Level(v), logging.vmodule.filter, false) - return nil -} - -// moduleSpec represents the setting of the -vmodule flag. -type moduleSpec struct { - filter []modulePat -} - -// modulePat contains a filter for the -vmodule flag. -// It holds a verbosity level and a file pattern to match. -type modulePat struct { - pattern string - literal bool // The pattern is a literal string - level Level -} - -// match reports whether the file matches the pattern. It uses a string -// comparison if the pattern contains no metacharacters. -func (m *modulePat) match(file string) bool { - if m.literal { - return file == m.pattern - } - match, _ := filepath.Match(m.pattern, file) - return match -} - -func (m *moduleSpec) String() string { - // Lock because the type is not atomic. TODO: clean this up. - logging.mu.Lock() - defer logging.mu.Unlock() - var b bytes.Buffer - for i, f := range m.filter { - if i > 0 { - b.WriteRune(',') - } - fmt.Fprintf(&b, "%s=%d", f.pattern, f.level) - } - return b.String() -} - -// Get is part of the (Go 1.2) flag.Getter interface. It always returns nil for this flag type since the -// struct is not exported. -func (m *moduleSpec) Get() interface{} { - return nil -} - -var errVmoduleSyntax = errors.New("syntax error: expect comma-separated list of filename=N") - -// Syntax: -vmodule=recordio=2,file=1,gfs*=3 -func (m *moduleSpec) Set(value string) error { - var filter []modulePat - for _, pat := range strings.Split(value, ",") { - if len(pat) == 0 { - // Empty strings such as from a trailing comma can be ignored. - continue - } - patLev := strings.Split(pat, "=") - if len(patLev) != 2 || len(patLev[0]) == 0 || len(patLev[1]) == 0 { - return errVmoduleSyntax - } - pattern := patLev[0] - v, err := strconv.Atoi(patLev[1]) - if err != nil { - return errors.New("syntax error: expect comma-separated list of filename=N") - } - if v < 0 { - return errors.New("negative value for vmodule level") - } - if v == 0 { - continue // Ignore. It's harmless but no point in paying the overhead. - } - // TODO: check syntax of filter? - filter = append(filter, modulePat{pattern, isLiteral(pattern), Level(v)}) - } - logging.mu.Lock() - defer logging.mu.Unlock() - logging.setVState(logging.verbosity, filter, true) - return nil -} - -// isLiteral reports whether the pattern is a literal string, that is, has no metacharacters -// that require filepath.Match to be called to match the pattern. -func isLiteral(pattern string) bool { - return !strings.ContainsAny(pattern, `\*?[]`) -} - -// traceLocation represents the setting of the -log_backtrace_at flag. -type traceLocation struct { - file string - line int -} - -// isSet reports whether the trace location has been specified. -// logging.mu is held. -func (t *traceLocation) isSet() bool { - return t.line > 0 -} - -// match reports whether the specified file and line matches the trace location. -// The argument file name is the full path, not the basename specified in the flag. -// logging.mu is held. -func (t *traceLocation) match(file string, line int) bool { - if t.line != line { - return false - } - if i := strings.LastIndex(file, "/"); i >= 0 { - file = file[i+1:] - } - return t.file == file -} - -func (t *traceLocation) String() string { - // Lock because the type is not atomic. TODO: clean this up. - logging.mu.Lock() - defer logging.mu.Unlock() - return fmt.Sprintf("%s:%d", t.file, t.line) -} - -// Get is part of the (Go 1.2) flag.Getter interface. It always returns nil for this flag type since the -// struct is not exported -func (t *traceLocation) Get() interface{} { - return nil -} - -var errTraceSyntax = errors.New("syntax error: expect file.go:234") - -// Syntax: -log_backtrace_at=gopherflakes.go:234 -// Note that unlike vmodule the file extension is included here. -func (t *traceLocation) Set(value string) error { - if value == "" { - // Unset. - t.line = 0 - t.file = "" - } - fields := strings.Split(value, ":") - if len(fields) != 2 { - return errTraceSyntax - } - file, line := fields[0], fields[1] - if !strings.Contains(file, ".") { - return errTraceSyntax - } - v, err := strconv.Atoi(line) - if err != nil { - return errTraceSyntax - } - if v <= 0 { - return errors.New("negative or zero value for level") - } - logging.mu.Lock() - defer logging.mu.Unlock() - t.line = v - t.file = file - return nil -} - -// flushSyncWriter is the interface satisfied by logging destinations. -type flushSyncWriter interface { - Flush() error - Sync() error - io.Writer -} - -func init() { - flag.BoolVar(&logging.toStderr, "logtostderr", false, "log to standard error instead of files") - flag.BoolVar(&logging.alsoToStderr, "alsologtostderr", false, "log to standard error as well as files") - flag.Var(&logging.verbosity, "v", "log level for V logs") - flag.Var(&logging.stderrThreshold, "stderrthreshold", "logs at or above this threshold go to stderr") - flag.Var(&logging.vmodule, "vmodule", "comma-separated list of pattern=N settings for file-filtered logging") - flag.Var(&logging.traceLocation, "log_backtrace_at", "when logging hits line file:N, emit a stack trace") - - // Default stderrThreshold is ERROR. - logging.stderrThreshold = errorLog - - logging.setVState(0, nil, false) - go logging.flushDaemon() -} - -// Flush flushes all pending log I/O. -func Flush() { - logging.lockAndFlushAll() -} - -// loggingT collects all the global state of the logging setup. -type loggingT struct { - // Boolean flags. Not handled atomically because the flag.Value interface - // does not let us avoid the =true, and that shorthand is necessary for - // compatibility. TODO: does this matter enough to fix? Seems unlikely. - toStderr bool // The -logtostderr flag. - alsoToStderr bool // The -alsologtostderr flag. - - // Level flag. Handled atomically. - stderrThreshold severity // The -stderrthreshold flag. - - // freeList is a list of byte buffers, maintained under freeListMu. - freeList *buffer - // freeListMu maintains the free list. It is separate from the main mutex - // so buffers can be grabbed and printed to without holding the main lock, - // for better parallelization. - freeListMu sync.Mutex - - // mu protects the remaining elements of this structure and is - // used to synchronize logging. - mu sync.Mutex - // file holds writer for each of the log types. - file [numSeverity]flushSyncWriter - // pcs is used in V to avoid an allocation when computing the caller's PC. - pcs [1]uintptr - // vmap is a cache of the V Level for each V() call site, identified by PC. - // It is wiped whenever the vmodule flag changes state. - vmap map[uintptr]Level - // filterLength stores the length of the vmodule filter chain. If greater - // than zero, it means vmodule is enabled. It may be read safely - // using sync.LoadInt32, but is only modified under mu. - filterLength int32 - // traceLocation is the state of the -log_backtrace_at flag. - traceLocation traceLocation - // These flags are modified only under lock, although verbosity may be fetched - // safely using atomic.LoadInt32. - vmodule moduleSpec // The state of the -vmodule flag. - verbosity Level // V logging level, the value of the -v flag/ -} - -// buffer holds a byte Buffer for reuse. The zero value is ready for use. -type buffer struct { - bytes.Buffer - tmp [64]byte // temporary byte array for creating headers. - next *buffer -} - -var logging loggingT - -// setVState sets a consistent state for V logging. -// l.mu is held. -func (l *loggingT) setVState(verbosity Level, filter []modulePat, setFilter bool) { - // Turn verbosity off so V will not fire while we are in transition. - logging.verbosity.set(0) - // Ditto for filter length. - atomic.StoreInt32(&logging.filterLength, 0) - - // Set the new filters and wipe the pc->Level map if the filter has changed. - if setFilter { - logging.vmodule.filter = filter - logging.vmap = make(map[uintptr]Level) - } - - // Things are consistent now, so enable filtering and verbosity. - // They are enabled in order opposite to that in V. - atomic.StoreInt32(&logging.filterLength, int32(len(filter))) - logging.verbosity.set(verbosity) -} - -// getBuffer returns a new, ready-to-use buffer. -func (l *loggingT) getBuffer() *buffer { - l.freeListMu.Lock() - b := l.freeList - if b != nil { - l.freeList = b.next - } - l.freeListMu.Unlock() - if b == nil { - b = new(buffer) - } else { - b.next = nil - b.Reset() - } - return b -} - -// putBuffer returns a buffer to the free list. -func (l *loggingT) putBuffer(b *buffer) { - if b.Len() >= 256 { - // Let big buffers die a natural death. - return - } - l.freeListMu.Lock() - b.next = l.freeList - l.freeList = b - l.freeListMu.Unlock() -} - -var timeNow = time.Now // Stubbed out for testing. - -/* -header formats a log header as defined by the C++ implementation. -It returns a buffer containing the formatted header and the user's file and line number. -The depth specifies how many stack frames above lives the source line to be identified in the log message. - -Log lines have this form: - Lmmdd hh:mm:ss.uuuuuu threadid file:line] msg... -where the fields are defined as follows: - L A single character, representing the log level (eg 'I' for INFO) - mm The month (zero padded; ie May is '05') - dd The day (zero padded) - hh:mm:ss.uuuuuu Time in hours, minutes and fractional seconds - threadid The space-padded thread ID as returned by GetTID() - file The file name - line The line number - msg The user-supplied message -*/ -func (l *loggingT) header(s severity, depth int) (*buffer, string, int) { - _, file, line, ok := runtime.Caller(3 + depth) - if !ok { - file = "???" - line = 1 - } else { - slash := strings.LastIndex(file, "/") - if slash >= 0 { - file = file[slash+1:] - } - } - return l.formatHeader(s, file, line), file, line -} - -// formatHeader formats a log header using the provided file name and line number. -func (l *loggingT) formatHeader(s severity, file string, line int) *buffer { - now := timeNow() - if line < 0 { - line = 0 // not a real line number, but acceptable to someDigits - } - if s > fatalLog { - s = infoLog // for safety. - } - buf := l.getBuffer() - - // Avoid Fprintf, for speed. The format is so simple that we can do it quickly by hand. - // It's worth about 3X. Fprintf is hard. - _, month, day := now.Date() - hour, minute, second := now.Clock() - // Lmmdd hh:mm:ss.uuuuuu threadid file:line] - buf.tmp[0] = severityChar[s] - buf.twoDigits(1, int(month)) - buf.twoDigits(3, day) - buf.tmp[5] = ' ' - buf.twoDigits(6, hour) - buf.tmp[8] = ':' - buf.twoDigits(9, minute) - buf.tmp[11] = ':' - buf.twoDigits(12, second) - buf.tmp[14] = '.' - buf.nDigits(6, 15, now.Nanosecond()/1000, '0') - buf.tmp[21] = ' ' - buf.nDigits(7, 22, pid, ' ') // TODO: should be TID - buf.tmp[29] = ' ' - buf.Write(buf.tmp[:30]) - buf.WriteString(file) - buf.tmp[0] = ':' - n := buf.someDigits(1, line) - buf.tmp[n+1] = ']' - buf.tmp[n+2] = ' ' - buf.Write(buf.tmp[:n+3]) - return buf -} - -// Some custom tiny helper functions to print the log header efficiently. - -const digits = "0123456789" - -// twoDigits formats a zero-prefixed two-digit integer at buf.tmp[i]. -func (buf *buffer) twoDigits(i, d int) { - buf.tmp[i+1] = digits[d%10] - d /= 10 - buf.tmp[i] = digits[d%10] -} - -// nDigits formats an n-digit integer at buf.tmp[i], -// padding with pad on the left. -// It assumes d >= 0. -func (buf *buffer) nDigits(n, i, d int, pad byte) { - j := n - 1 - for ; j >= 0 && d > 0; j-- { - buf.tmp[i+j] = digits[d%10] - d /= 10 - } - for ; j >= 0; j-- { - buf.tmp[i+j] = pad - } -} - -// someDigits formats a zero-prefixed variable-width integer at buf.tmp[i]. -func (buf *buffer) someDigits(i, d int) int { - // Print into the top, then copy down. We know there's space for at least - // a 10-digit number. - j := len(buf.tmp) - for { - j-- - buf.tmp[j] = digits[d%10] - d /= 10 - if d == 0 { - break - } - } - return copy(buf.tmp[i:], buf.tmp[j:]) -} - -func (l *loggingT) println(s severity, args ...interface{}) { - buf, file, line := l.header(s, 0) - fmt.Fprintln(buf, args...) - l.output(s, buf, file, line, false) -} - -func (l *loggingT) print(s severity, args ...interface{}) { - l.printDepth(s, 1, args...) -} - -func (l *loggingT) printDepth(s severity, depth int, args ...interface{}) { - buf, file, line := l.header(s, depth) - fmt.Fprint(buf, args...) - if buf.Bytes()[buf.Len()-1] != '\n' { - buf.WriteByte('\n') - } - l.output(s, buf, file, line, false) -} - -func (l *loggingT) printf(s severity, format string, args ...interface{}) { - buf, file, line := l.header(s, 0) - fmt.Fprintf(buf, format, args...) - if buf.Bytes()[buf.Len()-1] != '\n' { - buf.WriteByte('\n') - } - l.output(s, buf, file, line, false) -} - -// printWithFileLine behaves like print but uses the provided file and line number. If -// alsoLogToStderr is true, the log message always appears on standard error; it -// will also appear in the log file unless --logtostderr is set. -func (l *loggingT) printWithFileLine(s severity, file string, line int, alsoToStderr bool, args ...interface{}) { - buf := l.formatHeader(s, file, line) - fmt.Fprint(buf, args...) - if buf.Bytes()[buf.Len()-1] != '\n' { - buf.WriteByte('\n') - } - l.output(s, buf, file, line, alsoToStderr) -} - -// output writes the data to the log files and releases the buffer. -func (l *loggingT) output(s severity, buf *buffer, file string, line int, alsoToStderr bool) { - l.mu.Lock() - if l.traceLocation.isSet() { - if l.traceLocation.match(file, line) { - buf.Write(stacks(false)) - } - } - data := buf.Bytes() - if !flag.Parsed() { - os.Stderr.Write([]byte("ERROR: logging before flag.Parse: ")) - os.Stderr.Write(data) - } else if l.toStderr { - os.Stderr.Write(data) - } else { - if alsoToStderr || l.alsoToStderr || s >= l.stderrThreshold.get() { - os.Stderr.Write(data) - } - if l.file[s] == nil { - if err := l.createFiles(s); err != nil { - os.Stderr.Write(data) // Make sure the message appears somewhere. - l.exit(err) - } - } - switch s { - case fatalLog: - l.file[fatalLog].Write(data) - fallthrough - case errorLog: - l.file[errorLog].Write(data) - fallthrough - case warningLog: - l.file[warningLog].Write(data) - fallthrough - case infoLog: - l.file[infoLog].Write(data) - } - } - if s == fatalLog { - // If we got here via Exit rather than Fatal, print no stacks. - if atomic.LoadUint32(&fatalNoStacks) > 0 { - l.mu.Unlock() - timeoutFlush(10 * time.Second) - os.Exit(1) - } - // Dump all goroutine stacks before exiting. - // First, make sure we see the trace for the current goroutine on standard error. - // If -logtostderr has been specified, the loop below will do that anyway - // as the first stack in the full dump. - if !l.toStderr { - os.Stderr.Write(stacks(false)) - } - // Write the stack trace for all goroutines to the files. - trace := stacks(true) - logExitFunc = func(error) {} // If we get a write error, we'll still exit below. - for log := fatalLog; log >= infoLog; log-- { - if f := l.file[log]; f != nil { // Can be nil if -logtostderr is set. - f.Write(trace) - } - } - l.mu.Unlock() - timeoutFlush(10 * time.Second) - os.Exit(255) // C++ uses -1, which is silly because it's anded with 255 anyway. - } - l.putBuffer(buf) - l.mu.Unlock() - if stats := severityStats[s]; stats != nil { - atomic.AddInt64(&stats.lines, 1) - atomic.AddInt64(&stats.bytes, int64(len(data))) - } -} - -// timeoutFlush calls Flush and returns when it completes or after timeout -// elapses, whichever happens first. This is needed because the hooks invoked -// by Flush may deadlock when glog.Fatal is called from a hook that holds -// a lock. -func timeoutFlush(timeout time.Duration) { - done := make(chan bool, 1) - go func() { - Flush() // calls logging.lockAndFlushAll() - done <- true - }() - select { - case <-done: - case <-time.After(timeout): - fmt.Fprintln(os.Stderr, "glog: Flush took longer than", timeout) - } -} - -// stacks is a wrapper for runtime.Stack that attempts to recover the data for all goroutines. -func stacks(all bool) []byte { - // We don't know how big the traces are, so grow a few times if they don't fit. Start large, though. - n := 10000 - if all { - n = 100000 - } - var trace []byte - for i := 0; i < 5; i++ { - trace = make([]byte, n) - nbytes := runtime.Stack(trace, all) - if nbytes < len(trace) { - return trace[:nbytes] - } - n *= 2 - } - return trace -} - -// logExitFunc provides a simple mechanism to override the default behavior -// of exiting on error. Used in testing and to guarantee we reach a required exit -// for fatal logs. Instead, exit could be a function rather than a method but that -// would make its use clumsier. -var logExitFunc func(error) - -// exit is called if there is trouble creating or writing log files. -// It flushes the logs and exits the program; there's no point in hanging around. -// l.mu is held. -func (l *loggingT) exit(err error) { - fmt.Fprintf(os.Stderr, "log: exiting because of error: %s\n", err) - // If logExitFunc is set, we do that instead of exiting. - if logExitFunc != nil { - logExitFunc(err) - return - } - l.flushAll() - os.Exit(2) -} - -// syncBuffer joins a bufio.Writer to its underlying file, providing access to the -// file's Sync method and providing a wrapper for the Write method that provides log -// file rotation. There are conflicting methods, so the file cannot be embedded. -// l.mu is held for all its methods. -type syncBuffer struct { - logger *loggingT - *bufio.Writer - file *os.File - sev severity - nbytes uint64 // The number of bytes written to this file -} - -func (sb *syncBuffer) Sync() error { - return sb.file.Sync() -} - -func (sb *syncBuffer) Write(p []byte) (n int, err error) { - if sb.nbytes+uint64(len(p)) >= MaxSize { - if err := sb.rotateFile(time.Now()); err != nil { - sb.logger.exit(err) - } - } - n, err = sb.Writer.Write(p) - sb.nbytes += uint64(n) - if err != nil { - sb.logger.exit(err) - } - return -} - -// rotateFile closes the syncBuffer's file and starts a new one. -func (sb *syncBuffer) rotateFile(now time.Time) error { - if sb.file != nil { - sb.Flush() - sb.file.Close() - } - var err error - sb.file, _, err = create(severityName[sb.sev], now) - sb.nbytes = 0 - if err != nil { - return err - } - - sb.Writer = bufio.NewWriterSize(sb.file, bufferSize) - - // Write header. - var buf bytes.Buffer - fmt.Fprintf(&buf, "Log file created at: %s\n", now.Format("2006/01/02 15:04:05")) - fmt.Fprintf(&buf, "Running on machine: %s\n", host) - fmt.Fprintf(&buf, "Binary: Built with %s %s for %s/%s\n", runtime.Compiler, runtime.Version(), runtime.GOOS, runtime.GOARCH) - fmt.Fprintf(&buf, "Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg\n") - n, err := sb.file.Write(buf.Bytes()) - sb.nbytes += uint64(n) - return err -} - -// bufferSize sizes the buffer associated with each log file. It's large -// so that log records can accumulate without the logging thread blocking -// on disk I/O. The flushDaemon will block instead. -const bufferSize = 256 * 1024 - -// createFiles creates all the log files for severity from sev down to infoLog. -// l.mu is held. -func (l *loggingT) createFiles(sev severity) error { - now := time.Now() - // Files are created in decreasing severity order, so as soon as we find one - // has already been created, we can stop. - for s := sev; s >= infoLog && l.file[s] == nil; s-- { - sb := &syncBuffer{ - logger: l, - sev: s, - } - if err := sb.rotateFile(now); err != nil { - return err - } - l.file[s] = sb - } - return nil -} - -const flushInterval = 30 * time.Second - -// flushDaemon periodically flushes the log file buffers. -func (l *loggingT) flushDaemon() { - for _ = range time.NewTicker(flushInterval).C { - l.lockAndFlushAll() - } -} - -// lockAndFlushAll is like flushAll but locks l.mu first. -func (l *loggingT) lockAndFlushAll() { - l.mu.Lock() - l.flushAll() - l.mu.Unlock() -} - -// flushAll flushes all the logs and attempts to "sync" their data to disk. -// l.mu is held. -func (l *loggingT) flushAll() { - // Flush from fatal down, in case there's trouble flushing. - for s := fatalLog; s >= infoLog; s-- { - file := l.file[s] - if file != nil { - file.Flush() // ignore error - file.Sync() // ignore error - } - } -} - -// CopyStandardLogTo arranges for messages written to the Go "log" package's -// default logs to also appear in the Google logs for the named and lower -// severities. Subsequent changes to the standard log's default output location -// or format may break this behavior. -// -// Valid names are "INFO", "WARNING", "ERROR", and "FATAL". If the name is not -// recognized, CopyStandardLogTo panics. -func CopyStandardLogTo(name string) { - sev, ok := severityByName(name) - if !ok { - panic(fmt.Sprintf("log.CopyStandardLogTo(%q): unrecognized severity name", name)) - } - // Set a log format that captures the user's file and line: - // d.go:23: message - stdLog.SetFlags(stdLog.Lshortfile) - stdLog.SetOutput(logBridge(sev)) -} - -// logBridge provides the Write method that enables CopyStandardLogTo to connect -// Go's standard logs to the logs provided by this package. -type logBridge severity - -// Write parses the standard logging line and passes its components to the -// logger for severity(lb). -func (lb logBridge) Write(b []byte) (n int, err error) { - var ( - file = "???" - line = 1 - text string - ) - // Split "d.go:23: message" into "d.go", "23", and "message". - if parts := bytes.SplitN(b, []byte{':'}, 3); len(parts) != 3 || len(parts[0]) < 1 || len(parts[2]) < 1 { - text = fmt.Sprintf("bad log format: %s", b) - } else { - file = string(parts[0]) - text = string(parts[2][1:]) // skip leading space - line, err = strconv.Atoi(string(parts[1])) - if err != nil { - text = fmt.Sprintf("bad line number: %s", b) - line = 1 - } - } - // printWithFileLine with alsoToStderr=true, so standard log messages - // always appear on standard error. - logging.printWithFileLine(severity(lb), file, line, true, text) - return len(b), nil -} - -// setV computes and remembers the V level for a given PC -// when vmodule is enabled. -// File pattern matching takes the basename of the file, stripped -// of its .go suffix, and uses filepath.Match, which is a little more -// general than the *? matching used in C++. -// l.mu is held. -func (l *loggingT) setV(pc uintptr) Level { - fn := runtime.FuncForPC(pc) - file, _ := fn.FileLine(pc) - // The file is something like /a/b/c/d.go. We want just the d. - if strings.HasSuffix(file, ".go") { - file = file[:len(file)-3] - } - if slash := strings.LastIndex(file, "/"); slash >= 0 { - file = file[slash+1:] - } - for _, filter := range l.vmodule.filter { - if filter.match(file) { - l.vmap[pc] = filter.level - return filter.level - } - } - l.vmap[pc] = 0 - return 0 -} - -// Verbose is a boolean type that implements Infof (like Printf) etc. -// See the documentation of V for more information. -type Verbose bool - -// V reports whether verbosity at the call site is at least the requested level. -// The returned value is a boolean of type Verbose, which implements Info, Infoln -// and Infof. These methods will write to the Info log if called. -// Thus, one may write either -// if glog.V(2) { glog.Info("log this") } -// or -// glog.V(2).Info("log this") -// The second form is shorter but the first is cheaper if logging is off because it does -// not evaluate its arguments. -// -// Whether an individual call to V generates a log record depends on the setting of -// the -v and --vmodule flags; both are off by default. If the level in the call to -// V is at least the value of -v, or of -vmodule for the source file containing the -// call, the V call will log. -func V(level Level) Verbose { - // This function tries hard to be cheap unless there's work to do. - // The fast path is two atomic loads and compares. - - // Here is a cheap but safe test to see if V logging is enabled globally. - if logging.verbosity.get() >= level { - return Verbose(true) - } - - // It's off globally but it vmodule may still be set. - // Here is another cheap but safe test to see if vmodule is enabled. - if atomic.LoadInt32(&logging.filterLength) > 0 { - // Now we need a proper lock to use the logging structure. The pcs field - // is shared so we must lock before accessing it. This is fairly expensive, - // but if V logging is enabled we're slow anyway. - logging.mu.Lock() - defer logging.mu.Unlock() - if runtime.Callers(2, logging.pcs[:]) == 0 { - return Verbose(false) - } - v, ok := logging.vmap[logging.pcs[0]] - if !ok { - v = logging.setV(logging.pcs[0]) - } - return Verbose(v >= level) - } - return Verbose(false) -} - -// Info is equivalent to the global Info function, guarded by the value of v. -// See the documentation of V for usage. -func (v Verbose) Info(args ...interface{}) { - if v { - logging.print(infoLog, args...) - } -} - -// Infoln is equivalent to the global Infoln function, guarded by the value of v. -// See the documentation of V for usage. -func (v Verbose) Infoln(args ...interface{}) { - if v { - logging.println(infoLog, args...) - } -} - -// Infof is equivalent to the global Infof function, guarded by the value of v. -// See the documentation of V for usage. -func (v Verbose) Infof(format string, args ...interface{}) { - if v { - logging.printf(infoLog, format, args...) - } -} - -// Info logs to the INFO log. -// Arguments are handled in the manner of fmt.Print; a newline is appended if missing. -func Info(args ...interface{}) { - logging.print(infoLog, args...) -} - -// InfoDepth acts as Info but uses depth to determine which call frame to log. -// InfoDepth(0, "msg") is the same as Info("msg"). -func InfoDepth(depth int, args ...interface{}) { - logging.printDepth(infoLog, depth, args...) -} - -// Infoln logs to the INFO log. -// Arguments are handled in the manner of fmt.Println; a newline is appended if missing. -func Infoln(args ...interface{}) { - logging.println(infoLog, args...) -} - -// Infof logs to the INFO log. -// Arguments are handled in the manner of fmt.Printf; a newline is appended if missing. -func Infof(format string, args ...interface{}) { - logging.printf(infoLog, format, args...) -} - -// Warning logs to the WARNING and INFO logs. -// Arguments are handled in the manner of fmt.Print; a newline is appended if missing. -func Warning(args ...interface{}) { - logging.print(warningLog, args...) -} - -// WarningDepth acts as Warning but uses depth to determine which call frame to log. -// WarningDepth(0, "msg") is the same as Warning("msg"). -func WarningDepth(depth int, args ...interface{}) { - logging.printDepth(warningLog, depth, args...) -} - -// Warningln logs to the WARNING and INFO logs. -// Arguments are handled in the manner of fmt.Println; a newline is appended if missing. -func Warningln(args ...interface{}) { - logging.println(warningLog, args...) -} - -// Warningf logs to the WARNING and INFO logs. -// Arguments are handled in the manner of fmt.Printf; a newline is appended if missing. -func Warningf(format string, args ...interface{}) { - logging.printf(warningLog, format, args...) -} - -// Error logs to the ERROR, WARNING, and INFO logs. -// Arguments are handled in the manner of fmt.Print; a newline is appended if missing. -func Error(args ...interface{}) { - logging.print(errorLog, args...) -} - -// ErrorDepth acts as Error but uses depth to determine which call frame to log. -// ErrorDepth(0, "msg") is the same as Error("msg"). -func ErrorDepth(depth int, args ...interface{}) { - logging.printDepth(errorLog, depth, args...) -} - -// Errorln logs to the ERROR, WARNING, and INFO logs. -// Arguments are handled in the manner of fmt.Println; a newline is appended if missing. -func Errorln(args ...interface{}) { - logging.println(errorLog, args...) -} - -// Errorf logs to the ERROR, WARNING, and INFO logs. -// Arguments are handled in the manner of fmt.Printf; a newline is appended if missing. -func Errorf(format string, args ...interface{}) { - logging.printf(errorLog, format, args...) -} - -// Fatal logs to the FATAL, ERROR, WARNING, and INFO logs, -// including a stack trace of all running goroutines, then calls os.Exit(255). -// Arguments are handled in the manner of fmt.Print; a newline is appended if missing. -func Fatal(args ...interface{}) { - logging.print(fatalLog, args...) -} - -// FatalDepth acts as Fatal but uses depth to determine which call frame to log. -// FatalDepth(0, "msg") is the same as Fatal("msg"). -func FatalDepth(depth int, args ...interface{}) { - logging.printDepth(fatalLog, depth, args...) -} - -// Fatalln logs to the FATAL, ERROR, WARNING, and INFO logs, -// including a stack trace of all running goroutines, then calls os.Exit(255). -// Arguments are handled in the manner of fmt.Println; a newline is appended if missing. -func Fatalln(args ...interface{}) { - logging.println(fatalLog, args...) -} - -// Fatalf logs to the FATAL, ERROR, WARNING, and INFO logs, -// including a stack trace of all running goroutines, then calls os.Exit(255). -// Arguments are handled in the manner of fmt.Printf; a newline is appended if missing. -func Fatalf(format string, args ...interface{}) { - logging.printf(fatalLog, format, args...) -} - -// fatalNoStacks is non-zero if we are to exit without dumping goroutine stacks. -// It allows Exit and relatives to use the Fatal logs. -var fatalNoStacks uint32 - -// Exit logs to the FATAL, ERROR, WARNING, and INFO logs, then calls os.Exit(1). -// Arguments are handled in the manner of fmt.Print; a newline is appended if missing. -func Exit(args ...interface{}) { - atomic.StoreUint32(&fatalNoStacks, 1) - logging.print(fatalLog, args...) -} - -// ExitDepth acts as Exit but uses depth to determine which call frame to log. -// ExitDepth(0, "msg") is the same as Exit("msg"). -func ExitDepth(depth int, args ...interface{}) { - atomic.StoreUint32(&fatalNoStacks, 1) - logging.printDepth(fatalLog, depth, args...) -} - -// Exitln logs to the FATAL, ERROR, WARNING, and INFO logs, then calls os.Exit(1). -func Exitln(args ...interface{}) { - atomic.StoreUint32(&fatalNoStacks, 1) - logging.println(fatalLog, args...) -} - -// Exitf logs to the FATAL, ERROR, WARNING, and INFO logs, then calls os.Exit(1). -// Arguments are handled in the manner of fmt.Printf; a newline is appended if missing. -func Exitf(format string, args ...interface{}) { - atomic.StoreUint32(&fatalNoStacks, 1) - logging.printf(fatalLog, format, args...) -} diff --git a/vendor/github.com/golang/glog/glog_file.go b/vendor/github.com/golang/glog/glog_file.go deleted file mode 100644 index 65075d2811..0000000000 --- a/vendor/github.com/golang/glog/glog_file.go +++ /dev/null @@ -1,124 +0,0 @@ -// Go support for leveled logs, analogous to https://code.google.com/p/google-glog/ -// -// Copyright 2013 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// File I/O for logs. - -package glog - -import ( - "errors" - "flag" - "fmt" - "os" - "os/user" - "path/filepath" - "strings" - "sync" - "time" -) - -// MaxSize is the maximum size of a log file in bytes. -var MaxSize uint64 = 1024 * 1024 * 1800 - -// logDirs lists the candidate directories for new log files. -var logDirs []string - -// If non-empty, overrides the choice of directory in which to write logs. -// See createLogDirs for the full list of possible destinations. -var logDir = flag.String("log_dir", "", "If non-empty, write log files in this directory") - -func createLogDirs() { - if *logDir != "" { - logDirs = append(logDirs, *logDir) - } - logDirs = append(logDirs, os.TempDir()) -} - -var ( - pid = os.Getpid() - program = filepath.Base(os.Args[0]) - host = "unknownhost" - userName = "unknownuser" -) - -func init() { - h, err := os.Hostname() - if err == nil { - host = shortHostname(h) - } - - current, err := user.Current() - if err == nil { - userName = current.Username - } - - // Sanitize userName since it may contain filepath separators on Windows. - userName = strings.Replace(userName, `\`, "_", -1) -} - -// shortHostname returns its argument, truncating at the first period. -// For instance, given "www.google.com" it returns "www". -func shortHostname(hostname string) string { - if i := strings.Index(hostname, "."); i >= 0 { - return hostname[:i] - } - return hostname -} - -// logName returns a new log file name containing tag, with start time t, and -// the name for the symlink for tag. -func logName(tag string, t time.Time) (name, link string) { - name = fmt.Sprintf("%s.%s.%s.log.%s.%04d%02d%02d-%02d%02d%02d.%d", - program, - host, - userName, - tag, - t.Year(), - t.Month(), - t.Day(), - t.Hour(), - t.Minute(), - t.Second(), - pid) - return name, program + "." + tag -} - -var onceLogDirs sync.Once - -// create creates a new log file and returns the file and its filename, which -// contains tag ("INFO", "FATAL", etc.) and t. If the file is created -// successfully, create also attempts to update the symlink for that tag, ignoring -// errors. -func create(tag string, t time.Time) (f *os.File, filename string, err error) { - onceLogDirs.Do(createLogDirs) - if len(logDirs) == 0 { - return nil, "", errors.New("log: no log dirs") - } - name, link := logName(tag, t) - var lastErr error - for _, dir := range logDirs { - fname := filepath.Join(dir, name) - f, err := os.Create(fname) - if err == nil { - symlink := filepath.Join(dir, link) - os.Remove(symlink) // ignore err - os.Symlink(name, symlink) // ignore err - return f, fname, nil - } - lastErr = err - } - return nil, "", fmt.Errorf("log: cannot create log: %v", lastErr) -} diff --git a/vendor/golang.org/x/sync/AUTHORS b/vendor/golang.org/x/sync/AUTHORS new file mode 100644 index 0000000000..15167cd746 --- /dev/null +++ b/vendor/golang.org/x/sync/AUTHORS @@ -0,0 +1,3 @@ +# This source code refers to The Go Authors for copyright purposes. +# The master list of authors is in the main Go distribution, +# visible at http://tip.golang.org/AUTHORS. diff --git a/vendor/golang.org/x/sync/CONTRIBUTORS b/vendor/golang.org/x/sync/CONTRIBUTORS new file mode 100644 index 0000000000..1c4577e968 --- /dev/null +++ b/vendor/golang.org/x/sync/CONTRIBUTORS @@ -0,0 +1,3 @@ +# This source code was written by the Go contributors. +# The master list of contributors is in the main Go distribution, +# visible at http://tip.golang.org/CONTRIBUTORS. diff --git a/vendor/golang.org/x/sync/LICENSE b/vendor/golang.org/x/sync/LICENSE new file mode 100644 index 0000000000..6a66aea5ea --- /dev/null +++ b/vendor/golang.org/x/sync/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/golang.org/x/sync/PATENTS b/vendor/golang.org/x/sync/PATENTS new file mode 100644 index 0000000000..733099041f --- /dev/null +++ b/vendor/golang.org/x/sync/PATENTS @@ -0,0 +1,22 @@ +Additional IP Rights Grant (Patents) + +"This implementation" means the copyrightable works distributed by +Google as part of the Go project. + +Google hereby grants to You a perpetual, worldwide, non-exclusive, +no-charge, royalty-free, irrevocable (except as stated in this section) +patent license to make, have made, use, offer to sell, sell, import, +transfer and otherwise run, modify and propagate the contents of this +implementation of Go, where such license applies only to those patent +claims, both currently owned or controlled by Google and acquired in +the future, licensable by Google that are necessarily infringed by this +implementation of Go. This grant does not include claims that would be +infringed only as a consequence of further modification of this +implementation. If you or your agent or exclusive licensee institute or +order or agree to the institution of patent litigation against any +entity (including a cross-claim or counterclaim in a lawsuit) alleging +that this implementation of Go or any code incorporated within this +implementation of Go constitutes direct or contributory patent +infringement, or inducement of patent infringement, then any patent +rights granted to you under this License for this implementation of Go +shall terminate as of the date such litigation is filed. diff --git a/vendor/golang.org/x/sync/errgroup/errgroup.go b/vendor/golang.org/x/sync/errgroup/errgroup.go new file mode 100644 index 0000000000..9857fe53d3 --- /dev/null +++ b/vendor/golang.org/x/sync/errgroup/errgroup.go @@ -0,0 +1,66 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package errgroup provides synchronization, error propagation, and Context +// cancelation for groups of goroutines working on subtasks of a common task. +package errgroup + +import ( + "context" + "sync" +) + +// A Group is a collection of goroutines working on subtasks that are part of +// the same overall task. +// +// A zero Group is valid and does not cancel on error. +type Group struct { + cancel func() + + wg sync.WaitGroup + + errOnce sync.Once + err error +} + +// WithContext returns a new Group and an associated Context derived from ctx. +// +// The derived Context is canceled the first time a function passed to Go +// returns a non-nil error or the first time Wait returns, whichever occurs +// first. +func WithContext(ctx context.Context) (*Group, context.Context) { + ctx, cancel := context.WithCancel(ctx) + return &Group{cancel: cancel}, ctx +} + +// Wait blocks until all function calls from the Go method have returned, then +// returns the first non-nil error (if any) from them. +func (g *Group) Wait() error { + g.wg.Wait() + if g.cancel != nil { + g.cancel() + } + return g.err +} + +// Go calls the given function in a new goroutine. +// +// The first call to return a non-nil error cancels the group; its error will be +// returned by Wait. +func (g *Group) Go(f func() error) { + g.wg.Add(1) + + go func() { + defer g.wg.Done() + + if err := f(); err != nil { + g.errOnce.Do(func() { + g.err = err + if g.cancel != nil { + g.cancel() + } + }) + } + }() +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 050ad9c23f..d8b0255d06 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -78,8 +78,6 @@ github.com/gogo/protobuf/sortkeys # github.com/golang-migrate/migrate/v4 v4.6.2 github.com/golang-migrate/migrate/v4/source github.com/golang-migrate/migrate/v4/source/file -# github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b -github.com/golang/glog # github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff github.com/golang/groupcache/lru # github.com/golang/mock v1.3.1 @@ -228,6 +226,8 @@ golang.org/x/net/websocket # golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 golang.org/x/oauth2 golang.org/x/oauth2/internal +# golang.org/x/sync v0.0.0-20190423024810-112230192c58 +golang.org/x/sync/errgroup # golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb golang.org/x/sys/cpu golang.org/x/sys/unix