From a2ab3e06182d68847989b22e40fb690d3ce5e9c8 Mon Sep 17 00:00:00 2001 From: Evan Cordell Date: Tue, 27 Nov 2018 09:01:59 -0500 Subject: [PATCH] feat(e2e-bare): add utilities for running olm and e2e locally against a remote cluster --- Makefile | 11 ++- cmd/catalog/main.go | 13 ++- cmd/olm/main.go | 24 ++--- go.sum | 3 - pkg/controller/operators/catalog/operator.go | 42 ++++---- .../operators/catalog/operator_test.go | 5 +- pkg/controller/operators/olm/operator.go | 72 ++++++++------ pkg/controller/operators/olm/operator_test.go | 12 +-- pkg/controller/operators/olm/requirements.go | 12 +-- pkg/lib/operatorclient/client.go | 10 +- pkg/lib/queueinformer/queueinformer.go | 20 ++-- .../queueinformer/queueinformer_operator.go | 34 ++++--- pkg/package-server/provider/inmem.go | 15 +-- pkg/package-server/server/server.go | 2 +- scripts/build_bare.sh | 21 ++++ scripts/install_bare.sh | 33 +++++++ scripts/run_e2e_bare.sh | 44 +++++++++ scripts/run_e2e_local.sh | 2 +- test/e2e/catalog_e2e_test.go | 11 ++- test/e2e/csv_e2e_test.go | 2 +- test/e2e/e2e-bare-values.yaml | 38 +++++++ test/e2e/metrics_e2e_test.go | 6 +- test/e2e/setup_test.go | 99 +++++++++++++++++++ test/e2e/util_test.go | 40 +++----- test/log/.gitkeep | 0 vendor/modules.txt | 2 +- 26 files changed, 414 insertions(+), 159 deletions(-) create mode 100644 scripts/build_bare.sh create mode 100755 scripts/install_bare.sh create mode 100755 scripts/run_e2e_bare.sh create mode 100644 test/e2e/e2e-bare-values.yaml create mode 100644 test/e2e/setup_test.go create mode 100644 test/log/.gitkeep diff --git a/Makefile b/Makefile index 7a460737a56..af2b151278b 100644 --- a/Makefile +++ b/Makefile @@ -69,14 +69,23 @@ run-local-shift: . ./scripts/install_local.sh local build/resources rm -rf build +setup-bare: + . ./scripts/build_bare.sh + . ./scripts/package-release.sh 1.0.0-e2e test/e2e/resources test/e2e/e2e-bare-values.yaml + . ./scripts/install_bare.sh e2e test/e2e/resources + e2e: - export NAMESPACE=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) + export NAMESPACE=default go test ./test/e2e/... e2e-local: . ./scripts/build_local.sh . ./scripts/run_e2e_local.sh $(TEST) +e2e-bare: + . ./scripts/build_bare.sh + . ./scripts/run_e2e_bare.sh $(TEST) + e2e-local-shift: . ./scripts/build_local_shift.sh . ./scripts/run_e2e_local.sh $(TEST) diff --git a/cmd/catalog/main.go b/cmd/catalog/main.go index 53d26b60a4f..4eaf8c9e772 100644 --- a/cmd/catalog/main.go +++ b/cmd/catalog/main.go @@ -9,6 +9,7 @@ import ( "time" log "github.com/sirupsen/logrus" + "k8s.io/api/core/v1" "github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/catalog" "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/signals" @@ -58,6 +59,16 @@ func main() { log.SetLevel(log.DebugLevel) } + // `namespaces` will always contain at least one entry: if `*watchedNamespaces` is + // the empty string, the resulting array will be `[]string{""}`. + namespaces := strings.Split(*watchedNamespaces, ",") + for _, ns := range namespaces { + if ns == v1.NamespaceAll { + namespaces = []string{v1.NamespaceAll} + break + } + } + // Serve a health check. http.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) @@ -65,7 +76,7 @@ func main() { go http.ListenAndServe(":8080", nil) // Create a new instance of the operator. - catalogOperator, err := catalog.NewOperator(*kubeConfigPath, *wakeupInterval, *catalogNamespace, strings.Split(*watchedNamespaces, ",")...) + catalogOperator, err := catalog.NewOperator(*kubeConfigPath, log.New(), *wakeupInterval, *catalogNamespace, namespaces...) if err != nil { log.Panicf("error configuring operator: %s", err.Error()) } diff --git a/cmd/olm/main.go b/cmd/olm/main.go index a2b7d7b2177..4719e28b105 100644 --- a/cmd/olm/main.go +++ b/cmd/olm/main.go @@ -10,6 +10,7 @@ import ( "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" + "k8s.io/api/core/v1" "github.com/operator-framework/operator-lifecycle-manager/pkg/api/client" "github.com/operator-framework/operator-lifecycle-manager/pkg/controller/install" @@ -24,15 +25,6 @@ const ( defaultWakeupInterval = 5 * time.Minute ) -// helper function for required env vars -func envOrDie(varname, description string) string { - val := os.Getenv(varname) - if len(val) == 0 { - log.Fatalf("must set env %s - %s", varname, description) - } - return val -} - // config flags defined globally so that they appear on the test binary as well var ( kubeConfigPath = flag.String( @@ -44,7 +36,7 @@ var ( watchedNamespaces = flag.String( "watchedNamespaces", "", "comma separated list of namespaces for alm operator to watch. "+ "If not set, or set to the empty string (e.g. `-watchedNamespaces=\"\"`), "+ - "alm operator will watch all namespaces in the cluster.") + "olm operator will watch all namespaces in the cluster.") debug = flag.Bool( "debug", false, "use debug log level") @@ -79,6 +71,12 @@ func main() { // `namespaces` will always contain at least one entry: if `*watchedNamespaces` is // the empty string, the resulting array will be `[]string{""}`. namespaces := strings.Split(*watchedNamespaces, ",") + for _, ns := range namespaces { + if ns == v1.NamespaceAll { + namespaces = []string{v1.NamespaceAll} + break + } + } // Create a client for OLM crClient, err := client.NewClient(*kubeConfigPath) @@ -86,10 +84,12 @@ func main() { log.Fatalf("error configuring client: %s", err.Error()) } - opClient := operatorclient.NewClientFromConfig(*kubeConfigPath) + logger := log.New() + + opClient := operatorclient.NewClientFromConfig(*kubeConfigPath, logger) // Create a new instance of the operator. - operator, err := olm.NewOperator(crClient, opClient, &install.StrategyResolver{}, *wakeupInterval, namespaces) + operator, err := olm.NewOperator(logger, crClient, opClient, &install.StrategyResolver{}, *wakeupInterval, namespaces) if err != nil { log.Fatalf("error configuring operator: %s", err.Error()) diff --git a/go.sum b/go.sum index 8d543f28ad5..8b004aabaf0 100644 --- a/go.sum +++ b/go.sum @@ -214,14 +214,12 @@ k8s.io/api v0.0.0-20180904230853-4e7be11eab3f h1:DLRkv8Ps4Sdx8Srj+UtGisj4whV7v/H k8s.io/api v0.0.0-20180904230853-4e7be11eab3f/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/apiextensions-apiserver v0.0.0-20180905004947-16750353bf97 h1:s4lWWs6JN5kWVzk5bztddkr5kgO/cGIbqTDP+QttUeQ= k8s.io/apiextensions-apiserver v0.0.0-20180905004947-16750353bf97/go.mod h1:IxkesAMoaCRoLrPJdZNZUQp9NfZnzqaVzLhb2VEQzXE= -k8s.io/apiextensions-apiserver v0.0.0-20181110192823-2c43ee60e25b h1:O3KqnOdhludLAAHs7bvV7UpPYow3gqLqF5Junc5hbw8= k8s.io/apimachinery v0.0.0-20181026144827-8ee1a638bafa h1:i0EOpPFWExNx7efINILpw8LJeah7gakRl1zjvwVfjiI= k8s.io/apimachinery v0.0.0-20181026144827-8ee1a638bafa/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= k8s.io/apiserver v0.0.0-20181026151315-13cfe3978170 h1:CqI85nZvPaV+7JFono0nAOGOx2brocqefcOhDPVhHKI= k8s.io/apiserver v0.0.0-20181026151315-13cfe3978170/go.mod h1:6bqaTSOSJavUIXUtfaR9Os9JtTCm8ZqH2SUl2S60C4w= k8s.io/client-go v8.0.0+incompatible h1:2pUaSg2x6iEHr8cia6zmWhoCXG1EDG9TCx9s//Aq7HY= k8s.io/client-go v8.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= -k8s.io/client-go v9.0.0+incompatible h1:2kqW3X2xQ9SbFvWZjGEHBLlWc1LG9JIJNXWkuqwdZ3A= k8s.io/code-generator v0.0.0-20180904193909-8c97d6ab64da h1:L6YB6ObZIbZlYikTQcCjzZGilwS3OVyQBA2esULs8VM= k8s.io/code-generator v0.0.0-20180904193909-8c97d6ab64da/go.mod h1:MYiN+ZJZ9HkETbgVZdWw2AsuAi9PZ4V80cwfuf2axe8= k8s.io/gengo v0.0.0-20181106084056-51747d6e00da h1:ZMvcXtMVbhUCtCuiSEzBV+Eur4swzfdxx6ZyX3qT6dk= @@ -230,7 +228,6 @@ k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92 h1:PgoMI/L1Nu5Vmvgm+vGheLuxKST8h6 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/kube-aggregator v0.0.0-20180905000155-efa32eb095fe h1:LM48rywzVEPRg+Os2oUL9/vsztPQGoxmiD3m5VySchw= k8s.io/kube-aggregator v0.0.0-20180905000155-efa32eb095fe/go.mod h1:8sbzT4QQKDEmSCIbfqjV0sd97GpUT7A4W626sBiYJmU= -k8s.io/kube-aggregator v0.0.0-20181110192014-6f96af33cb59 h1:8VFjmCurXo3sMW0ASrUvoE4aT8FCzsIz55uic1EyUIc= k8s.io/kube-openapi v0.0.0-20181031203759-72693cb1fadd h1:ggv/Vfza0i5xuhUZyYyxcc25AmQvHY8Zi1C2m8WgBvA= k8s.io/kube-openapi v0.0.0-20181031203759-72693cb1fadd/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/kubernetes v1.11.5-beta.0.0.20181108064615-3290824d1c7b h1:Ej2VgtycDFPfQunFFfAsWvdjT+dhte1CcwnPNId+V1k= diff --git a/pkg/controller/operators/catalog/operator.go b/pkg/controller/operators/catalog/operator.go index 3b5e8ec9afb..6c0826c9231 100644 --- a/pkg/controller/operators/catalog/operator.go +++ b/pkg/controller/operators/catalog/operator.go @@ -7,7 +7,7 @@ import ( "sync" "time" - log "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" v1beta1ext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" @@ -55,7 +55,7 @@ type Operator struct { } // NewOperator creates a new Catalog Operator. -func NewOperator(kubeconfigPath string, wakeupInterval time.Duration, operatorNamespace string, watchedNamespaces ...string) (*Operator, error) { +func NewOperator(kubeconfigPath string, logger *logrus.Logger, wakeupInterval time.Duration, operatorNamespace string, watchedNamespaces ...string) (*Operator, error) { // Default to watching all namespaces. if watchedNamespaces == nil { watchedNamespaces = []string{metav1.NamespaceAll} @@ -84,7 +84,7 @@ func NewOperator(kubeconfigPath string, wakeupInterval time.Duration, operatorNa } // Create a new queueinformer-based operator. - queueOperator, err := queueinformer.NewOperator(kubeconfigPath) + queueOperator, err := queueinformer.NewOperator(kubeconfigPath, logger) if err != nil { return nil, err } @@ -107,6 +107,7 @@ func NewOperator(kubeconfigPath string, wakeupInterval time.Duration, operatorNa nil, "catsrc", metrics.NewMetricsCatalogSource(op.client), + logger, ) for _, informer := range catsrcQueueInformer { op.RegisterQueueInformer(informer) @@ -121,6 +122,7 @@ func NewOperator(kubeconfigPath string, wakeupInterval time.Duration, operatorNa nil, "installplan", metrics.NewMetricsInstallPlan(op.client), + logger, ) for _, informer := range ipQueueInformers { op.RegisterQueueInformer(informer) @@ -135,6 +137,7 @@ func NewOperator(kubeconfigPath string, wakeupInterval time.Duration, operatorNa nil, "subscription", metrics.NewMetricsSubscription(op.client), + logger, ) op.subQueue = subscriptionQueue for _, informer := range subscriptionQueueInformers { @@ -147,7 +150,7 @@ func NewOperator(kubeconfigPath string, wakeupInterval time.Duration, operatorNa func (o *Operator) syncCatalogSources(obj interface{}) (syncError error) { catsrc, ok := obj.(*v1alpha1.CatalogSource) if !ok { - log.Debugf("wrong type: %#v", obj) + o.Log.Debugf("wrong type: %#v", obj) return fmt.Errorf("casting CatalogSource failed") } @@ -198,11 +201,11 @@ func (o *Operator) syncCatalogSources(obj interface{}) (syncError error) { func (o *Operator) syncSubscriptions(obj interface{}) (syncError error) { sub, ok := obj.(*v1alpha1.Subscription) if !ok { - log.Debugf("wrong type: %#v", obj) + o.Log.Debugf("wrong type: %#v", obj) return fmt.Errorf("casting Subscription failed") } - logger := log.WithFields(log.Fields{ + logger := o.Log.WithFields(logrus.Fields{ "sub": sub.GetName(), "namespace": sub.GetNamespace(), "source": sub.Spec.CatalogSource, @@ -248,18 +251,18 @@ func (o *Operator) requeueInstallPlan(name, namespace string) { func (o *Operator) syncInstallPlans(obj interface{}) (syncError error) { plan, ok := obj.(*v1alpha1.InstallPlan) if !ok { - log.Debugf("wrong type: %#v", obj) + o.Log.Debugf("wrong type: %#v", obj) return fmt.Errorf("casting InstallPlan failed") } - logger := log.WithFields(log.Fields{ + logger := o.Log.WithFields(logrus.Fields{ "ip": plan.GetName(), "namespace": plan.GetNamespace(), "phase": plan.Status.Phase, }) logger.Info("syncing") - outInstallPlan, syncError := transitionInstallPlanState(o, *plan) + outInstallPlan, syncError := transitionInstallPlanState(logger.Logger, o, *plan) if syncError != nil { logger = logger.WithField("syncError", syncError) @@ -297,23 +300,17 @@ type installPlanTransitioner interface { var _ installPlanTransitioner = &Operator{} -func transitionInstallPlanState(transitioner installPlanTransitioner, in v1alpha1.InstallPlan) (*v1alpha1.InstallPlan, error) { - logger := log.WithFields(log.Fields{ - "ip": in.GetName(), - "namespace": in.GetNamespace(), - "phase": in.Status.Phase, - }) - +func transitionInstallPlanState(log *logrus.Logger, transitioner installPlanTransitioner, in v1alpha1.InstallPlan) (*v1alpha1.InstallPlan, error) { out := in.DeepCopy() switch in.Status.Phase { case v1alpha1.InstallPlanPhaseNone: - logger.Debugf("setting phase to %s", v1alpha1.InstallPlanPhasePlanning) + log.Debugf("setting phase to %s", v1alpha1.InstallPlanPhasePlanning) out.Status.Phase = v1alpha1.InstallPlanPhasePlanning return out, nil case v1alpha1.InstallPlanPhasePlanning: - logger.Debug("attempting to resolve") + log.Debug("attempting to resolve") if err := transitioner.ResolvePlan(out); err != nil { out.Status.SetCondition(v1alpha1.ConditionFailed(v1alpha1.InstallPlanResolved, v1alpha1.InstallPlanReasonInstallCheckFailed, err)) @@ -331,15 +328,15 @@ func transitionInstallPlanState(transitioner installPlanTransitioner, in v1alpha case v1alpha1.InstallPlanPhaseRequiresApproval: if out.Spec.Approved { - logger.Debugf("approved, setting to %s", v1alpha1.InstallPlanPhasePlanning) + log.Debugf("approved, setting to %s", v1alpha1.InstallPlanPhasePlanning) out.Status.Phase = v1alpha1.InstallPlanPhaseInstalling } else { - logger.Debug("not approved, skipping sync") + log.Debug("not approved, skipping sync") } return out, nil case v1alpha1.InstallPlanPhaseInstalling: - logger.Debug("attempting to install") + log.Debug("attempting to install") if err := transitioner.ExecutePlan(out); err != nil { out.Status.SetCondition(v1alpha1.ConditionFailed(v1alpha1.InstallPlanInstalled, v1alpha1.InstallPlanReasonComponentFailed, err)) @@ -447,8 +444,7 @@ func (o *Operator) ExecutePlan(plan *v1alpha1.InstallPlan) error { continue case v1alpha1.StepStatusUnknown, v1alpha1.StepStatusNotPresent: - log.Debugf("resource kind: %s", step.Resource.Kind) - log.Debugf("resource name: %s", step.Resource.Name) + o.Log.WithFields(logrus.Fields{"kind": step.Resource.Kind, "name": step.Resource.Name}).Debug("execute resource") switch step.Resource.Kind { case crdKind: // Marshal the manifest into a CRD instance. diff --git a/pkg/controller/operators/catalog/operator_test.go b/pkg/controller/operators/catalog/operator_test.go index 874c48bf063..93ea975ccf0 100644 --- a/pkg/controller/operators/catalog/operator_test.go +++ b/pkg/controller/operators/catalog/operator_test.go @@ -2,6 +2,7 @@ package catalog import ( "errors" + "github.com/sirupsen/logrus" "testing" "github.com/ghodss/yaml" @@ -110,7 +111,7 @@ func TestTransitionInstallPlan(t *testing.T) { transitioner := &mockTransitioner{tt.transError} // Attempt to transition phases. - out, _ := transitionInstallPlanState(transitioner, *plan) + out, _ := transitionInstallPlanState(logrus.New(), transitioner, *plan) // Assert that the final phase is as expected. require.Equal(t, tt.expected, out.Status.Phase) @@ -386,7 +387,7 @@ func NewFakeOperator(clientObjs []runtime.Object, k8sObjs []runtime.Object, extO } // Create the new operator - queueOperator, err := queueinformer.NewOperatorFromClient(opClientFake) + queueOperator, err := queueinformer.NewOperatorFromClient(opClientFake, logrus.New()) op := &Operator{ Operator: queueOperator, client: clientFake, diff --git a/pkg/controller/operators/olm/operator.go b/pkg/controller/operators/olm/operator.go index e495163cedc..4c4715d1bed 100644 --- a/pkg/controller/operators/olm/operator.go +++ b/pkg/controller/operators/olm/operator.go @@ -6,8 +6,8 @@ import ( "strings" "time" - log "github.com/sirupsen/logrus" - aextv1beta1 "k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions" + "github.com/sirupsen/logrus" + extv1beta1 "k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -54,7 +54,7 @@ type Operator struct { recorder record.EventRecorder } -func NewOperator(crClient versioned.Interface, opClient operatorclient.ClientInterface, resolver install.StrategyResolverInterface, wakeupInterval time.Duration, namespaces []string) (*Operator, error) { +func NewOperator(logger *logrus.Logger, crClient versioned.Interface, opClient operatorclient.ClientInterface, resolver install.StrategyResolverInterface, wakeupInterval time.Duration, namespaces []string) (*Operator, error) { if wakeupInterval < 0 { wakeupInterval = FallbackWakeupInterval } @@ -62,7 +62,7 @@ func NewOperator(crClient versioned.Interface, opClient operatorclient.ClientInt namespaces = []string{metav1.NamespaceAll} } - queueOperator, err := queueinformer.NewOperatorFromClient(opClient) + queueOperator, err := queueinformer.NewOperatorFromClient(opClient, logger) if err != nil { return nil, err } @@ -89,6 +89,7 @@ func NewOperator(crClient versioned.Interface, opClient operatorclient.ClientInt nil, "roles", metrics.NewMetricsNil(), + logger, ) op.RegisterQueueInformer(roleQueueInformer) op.lister.RbacV1().RegisterRoleLister(metav1.NamespaceAll, roleInformer.Lister()) @@ -101,6 +102,7 @@ func NewOperator(crClient versioned.Interface, opClient operatorclient.ClientInt nil, "rolebindings", metrics.NewMetricsNil(), + logger, ) op.RegisterQueueInformer(roleBindingQueueInformer) op.lister.RbacV1().RegisterRoleBindingLister(metav1.NamespaceAll, roleBindingInformer.Lister()) @@ -113,6 +115,7 @@ func NewOperator(crClient versioned.Interface, opClient operatorclient.ClientInt nil, "clusterroles", metrics.NewMetricsNil(), + logger, ) op.RegisterQueueInformer(clusterRoleQueueInformer) op.lister.RbacV1().RegisterClusterRoleLister(clusterRoleInformer.Lister()) @@ -125,6 +128,7 @@ func NewOperator(crClient versioned.Interface, opClient operatorclient.ClientInt nil, "clusterrolebindings", metrics.NewMetricsNil(), + logger, ) op.lister.RbacV1().RegisterClusterRoleBindingLister(clusterRoleBindingInformer.Lister()) op.RegisterQueueInformer(clusterRoleBindingQueueInformer) @@ -138,6 +142,7 @@ func NewOperator(crClient versioned.Interface, opClient operatorclient.ClientInt nil, "namespaces", metrics.NewMetricsNil(), + logger, ) op.RegisterQueueInformer(namespaceQueueInformer) op.lister.CoreV1().RegisterNamespaceLister(namespaceInformer.Lister()) @@ -153,11 +158,12 @@ func NewOperator(crClient versioned.Interface, opClient operatorclient.ClientInt }, "apiservices", metrics.NewMetricsNil(), + logger, )) op.lister.APIRegistrationV1().RegisterAPIServiceLister(apiServiceInformer.Lister()) // Register CustomResourceDefinition QueueInformer - customResourceDefinitionInformer := aextv1beta1.NewSharedInformerFactory(opClient.ApiextensionsV1beta1Interface(), wakeupInterval).Apiextensions().V1beta1().CustomResourceDefinitions() + customResourceDefinitionInformer := extv1beta1.NewSharedInformerFactory(opClient.ApiextensionsV1beta1Interface(), wakeupInterval).Apiextensions().V1beta1().CustomResourceDefinitions() op.RegisterQueueInformer(queueinformer.NewInformer( workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "customresourcedefinitions"), customResourceDefinitionInformer.Informer(), @@ -167,6 +173,7 @@ func NewOperator(crClient versioned.Interface, opClient operatorclient.ClientInt }, "customresourcedefinitions", metrics.NewMetricsNil(), + logger, )) op.lister.APIExtensionsV1beta1().RegisterCustomResourceDefinitionLister(customResourceDefinitionInformer.Lister()) @@ -181,6 +188,7 @@ func NewOperator(crClient versioned.Interface, opClient operatorclient.ClientInt }, "secrets", metrics.NewMetricsNil(), + logger, )) op.lister.CoreV1().RegisterSecretLister(metav1.NamespaceAll, secretInformer.Lister()) @@ -195,6 +203,7 @@ func NewOperator(crClient versioned.Interface, opClient operatorclient.ClientInt }, "services", metrics.NewMetricsNil(), + logger, )) op.lister.CoreV1().RegisterServiceLister(metav1.NamespaceAll, serviceInformer.Lister()) @@ -209,6 +218,7 @@ func NewOperator(crClient versioned.Interface, opClient operatorclient.ClientInt }, "serviceaccounts", metrics.NewMetricsNil(), + logger, )) op.lister.CoreV1().RegisterServiceAccountLister(metav1.NamespaceAll, serviceAccountInformer.Lister()) @@ -217,7 +227,7 @@ func NewOperator(crClient versioned.Interface, opClient operatorclient.ClientInt DeleteFunc: op.deleteClusterServiceVersion, } for _, namespace := range namespaces { - log.WithField("namespace", namespace).Infof("watching CSVs") + logger.WithField("namespace", namespace).Infof("watching CSVs") sharedInformerFactory := externalversions.NewSharedInformerFactoryWithOptions(crClient, wakeupInterval, externalversions.WithNamespace(namespace)) csvInformer := sharedInformerFactory.Operators().V1alpha1().ClusterServiceVersions() op.lister.OperatorsV1alpha1().RegisterClusterServiceVersionLister(namespace, csvInformer.Lister()) @@ -225,7 +235,7 @@ func NewOperator(crClient versioned.Interface, opClient operatorclient.ClientInt // Register queue and QueueInformer queueName := fmt.Sprintf("%s/clusterserviceversions", namespace) csvQueue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), queueName) - csvQueueInformer := queueinformer.NewInformer(csvQueue, csvInformer.Informer(), op.syncClusterServiceVersion, csvHandlers, queueName, metrics.NewMetricsCSV(op.lister.OperatorsV1alpha1().ClusterServiceVersionLister())) + csvQueueInformer := queueinformer.NewInformer(csvQueue, csvInformer.Informer(), op.syncClusterServiceVersion, csvHandlers, queueName, metrics.NewMetricsCSV(op.lister.OperatorsV1alpha1().ClusterServiceVersionLister()), logger) op.RegisterQueueInformer(csvQueueInformer) op.csvQueues[namespace] = csvQueue } @@ -235,20 +245,20 @@ func NewOperator(crClient versioned.Interface, opClient operatorclient.ClientInt DeleteFunc: op.handleDeletion, } for _, namespace := range namespaces { - log.WithField("namespace", namespace).Infof("watching deployments") + logger.WithField("namespace", namespace).Infof("watching deployments") depInformer := informers.NewSharedInformerFactoryWithOptions(opClient.KubernetesInterface(), wakeupInterval, informers.WithNamespace(namespace)).Apps().V1().Deployments() op.lister.AppsV1().RegisterDeploymentLister(namespace, depInformer.Lister()) // Register queue and QueueInformer queueName := fmt.Sprintf("%s/csv-deployments", namespace) depQueue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), queueName) - depQueueInformer := queueinformer.NewInformer(depQueue, depInformer.Informer(), op.syncObject, depHandlers, queueName, metrics.NewMetricsNil()) + depQueueInformer := queueinformer.NewInformer(depQueue, depInformer.Informer(), op.syncObject, depHandlers, queueName, metrics.NewMetricsNil(), logger) op.RegisterQueueInformer(depQueueInformer) } // Create an informer for the operator group for _, namespace := range namespaces { - log.WithField("namespace", namespace).Infof("watching OperatorGroups") + logger.WithField("namespace", namespace).Infof("watching OperatorGroups") sharedInformerFactory := externalversions.NewSharedInformerFactoryWithOptions(crClient, wakeupInterval, externalversions.WithNamespace(namespace)) operatorGroupInformer := sharedInformerFactory.Operators().V1alpha2().OperatorGroups() op.lister.OperatorsV1alpha2().RegisterOperatorGroupLister(namespace, operatorGroupInformer.Lister()) @@ -256,7 +266,7 @@ func NewOperator(crClient versioned.Interface, opClient operatorclient.ClientInt // Register queue and QueueInformer queueName := fmt.Sprintf("%s/operatorgroups", namespace) operatorGroupQueue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), queueName) - operatorGroupQueueInformer := queueinformer.NewInformer(operatorGroupQueue, operatorGroupInformer.Informer(), op.syncOperatorGroups, nil, queueName, metrics.NewMetricsNil()) + operatorGroupQueueInformer := queueinformer.NewInformer(operatorGroupQueue, operatorGroupInformer.Informer(), op.syncOperatorGroups, nil, queueName, metrics.NewMetricsNil(), logger) op.RegisterQueueInformer(operatorGroupQueueInformer) } @@ -266,7 +276,7 @@ func NewOperator(crClient versioned.Interface, opClient operatorclient.ClientInt func (a *Operator) requeueCSV(name, namespace string) { // We can build the key directly, will need to change if queue uses different key scheme key := fmt.Sprintf("%s/%s", namespace, name) - logger := log.WithField("key", key) + logger := a.Log.WithField("key", key) logger.Debug("requeueing CSV") if queue, ok := a.csvQueues[metav1.NamespaceAll]; len(a.csvQueues) == 1 && ok { @@ -287,12 +297,12 @@ func (a *Operator) syncObject(obj interface{}) (syncError error) { runtimeObj, ok := obj.(runtime.Object) if !ok { syncError = errors.New("object sync: casting to runtime.Object failed") - log.Warn(syncError.Error()) + a.Log.Warn(syncError.Error()) return } gvk := runtimeObj.GetObjectKind().GroupVersionKind() - logger := log.WithFields(log.Fields{ + logger := a.Log.WithFields(logrus.Fields{ "group": gvk.Group, "version": gvk.Version, "kind": gvk.Kind, @@ -305,7 +315,7 @@ func (a *Operator) syncObject(obj interface{}) (syncError error) { logger.Warn(syncError.Error()) return } - logger = logger.WithFields(log.Fields{ + logger = a.Log.WithFields(logrus.Fields{ "name": metaObj.GetName(), "namespace": metaObj.GetNamespace(), }) @@ -324,11 +334,11 @@ func (a *Operator) syncObject(obj interface{}) (syncError error) { func (a *Operator) deleteClusterServiceVersion(obj interface{}) { clusterServiceVersion, ok := obj.(*v1alpha1.ClusterServiceVersion) if !ok { - log.Debugf("wrong type: %#v", obj) + a.Log.Debugf("wrong type: %#v", obj) return } - logger := log.WithFields(log.Fields{ + logger := a.Log.WithFields(logrus.Fields{ "csv": clusterServiceVersion.GetName(), "namespace": clusterServiceVersion.GetNamespace(), "phase": clusterServiceVersion.Status.Phase, @@ -358,7 +368,7 @@ func (a *Operator) deleteClusterServiceVersion(obj interface{}) { } func (a *Operator) removeDanglingChildCSVs(csv *v1alpha1.ClusterServiceVersion) error { - logger := log.WithFields(log.Fields{ + logger := a.Log.WithFields(logrus.Fields{ "csv": csv.GetName(), "namespace": csv.GetNamespace(), "phase": csv.Status.Phase, @@ -384,10 +394,10 @@ func (a *Operator) removeDanglingChildCSVs(csv *v1alpha1.ClusterServiceVersion) func (a *Operator) syncClusterServiceVersion(obj interface{}) (syncError error) { clusterServiceVersion, ok := obj.(*v1alpha1.ClusterServiceVersion) if !ok { - log.Debugf("wrong type: %#v", obj) + a.Log.Debugf("wrong type: %#v", obj) return fmt.Errorf("casting ClusterServiceVersion failed") } - logger := log.WithFields(log.Fields{ + logger := a.Log.WithFields(logrus.Fields{ "csv": clusterServiceVersion.GetName(), "namespace": clusterServiceVersion.GetNamespace(), "phase": clusterServiceVersion.Status.Phase, @@ -424,7 +434,7 @@ func (a *Operator) syncClusterServiceVersion(obj interface{}) (syncError error) // transitionCSVState moves the CSV status state machine along based on the current value and the current cluster state. func (a *Operator) transitionCSVState(in v1alpha1.ClusterServiceVersion) (out *v1alpha1.ClusterServiceVersion, syncError error) { - logger := log.WithFields(log.Fields{ + logger := a.Log.WithFields(logrus.Fields{ "csv": in.GetName(), "namespace": in.GetNamespace(), "phase": in.Status.Phase, @@ -623,14 +633,14 @@ func (a *Operator) findIntermediatesForDeletion(csv *v1alpha1.ClusterServiceVers next := a.isBeingReplaced(current, csvsInNamespace) for next != nil { csvs = append(csvs, current) - log.Debugf("checking to see if %s is running so we can delete %s", next.GetName(), csv.GetName()) + a.Log.Debugf("checking to see if %s is running so we can delete %s", next.GetName(), csv.GetName()) installer, nextStrategy, currentStrategy := a.parseStrategiesAndUpdateStatus(next) if nextStrategy == nil { - log.Debugf("couldn't get strategy for %s", next.GetName()) + a.Log.Debugf("couldn't get strategy for %s", next.GetName()) continue } if currentStrategy == nil { - log.Debugf("couldn't get strategy for %s", next.GetName()) + a.Log.Debugf("couldn't get strategy for %s", next.GetName()) continue } installed, _ := installer.CheckInstalled(nextStrategy) @@ -648,7 +658,7 @@ func (a *Operator) csvSet(namespace string) map[string]*v1alpha1.ClusterServiceV csvsInNamespace, err := a.lister.OperatorsV1alpha1().ClusterServiceVersionLister().ClusterServiceVersions(namespace).List(labels.Everything()) if err != nil { - log.Warnf("could not list CSVs while constructing CSV set") + a.Log.Warnf("could not list CSVs while constructing CSV set") return nil } @@ -665,7 +675,7 @@ func (a *Operator) checkReplacementsAndUpdateStatus(csv *v1alpha1.ClusterService return nil } if replacement := a.isBeingReplaced(csv, a.csvSet(csv.GetNamespace())); replacement != nil { - log.Infof("newer ClusterServiceVersion replacing %s, no-op", csv.SelfLink) + a.Log.Infof("newer ClusterServiceVersion replacing %s, no-op", csv.SelfLink) msg := fmt.Sprintf("being replaced by csv: %s", replacement.SelfLink) csv.SetPhase(v1alpha1.CSVPhaseReplacing, v1alpha1.CSVReasonBeingReplaced, msg, timeNow()) metrics.CSVUpgradeCount.Inc() @@ -781,9 +791,9 @@ func (a *Operator) apiServiceOwnerConflicts(in *v1alpha1.ClusterServiceVersion, func (a *Operator) isBeingReplaced(in *v1alpha1.ClusterServiceVersion, csvsInNamespace map[string]*v1alpha1.ClusterServiceVersion) (replacedBy *v1alpha1.ClusterServiceVersion) { for _, csv := range csvsInNamespace { - log.Infof("checking %s", csv.GetName()) + a.Log.Infof("checking %s", csv.GetName()) if csv.Spec.Replaces == in.GetName() { - log.Infof("%s replaced by %s", in.GetName(), csv.GetName()) + a.Log.Infof("%s replaced by %s", in.GetName(), csv.GetName()) replacedBy = csv.DeepCopy() return } @@ -792,13 +802,13 @@ func (a *Operator) isBeingReplaced(in *v1alpha1.ClusterServiceVersion, csvsInNam } func (a *Operator) isReplacing(in *v1alpha1.ClusterServiceVersion) *v1alpha1.ClusterServiceVersion { - log.Debugf("checking if csv is replacing an older version") + a.Log.Debugf("checking if csv is replacing an older version") if in.Spec.Replaces == "" { return nil } previous, err := a.lister.OperatorsV1alpha1().ClusterServiceVersionLister().ClusterServiceVersions(in.GetNamespace()).Get(in.Spec.Replaces) if err != nil { - log.Debugf("unable to get previous csv: %s", err.Error()) + a.Log.Debugf("unable to get previous csv: %s", err.Error()) return nil } return previous @@ -824,7 +834,7 @@ func (a *Operator) handleDeletion(obj interface{}) { } func (a *Operator) requeueOwnerCSVs(ownee metav1.Object) { - logger := log.WithFields(log.Fields{ + logger := a.Log.WithFields(logrus.Fields{ "ownee": ownee.GetName(), "selflink": ownee.GetSelfLink(), "namespace": ownee.GetNamespace(), diff --git a/pkg/controller/operators/olm/operator_test.go b/pkg/controller/operators/olm/operator_test.go index d7bf27d7cd2..14043f308a2 100644 --- a/pkg/controller/operators/olm/operator_test.go +++ b/pkg/controller/operators/olm/operator_test.go @@ -14,7 +14,7 @@ import ( "testing" "time" - log "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" @@ -131,7 +131,7 @@ func NewFakeOperator(clientObjs []runtime.Object, k8sObjs []runtime.Object, extO } // Create the new operator - queueOperator, err := queueinformer.NewOperatorFromClient(opClientFake) + queueOperator, err := queueinformer.NewOperatorFromClient(opClientFake, logrus.New()) op := &Operator{ Operator: queueOperator, client: clientFake, @@ -604,7 +604,7 @@ func generateCA(notAfter time.Time, organization string) (*certs.KeyPair, error) } func TestTransitionCSV(t *testing.T) { - log.SetLevel(log.DebugLevel) + logrus.SetLevel(logrus.DebugLevel) namespace := "ns" // Generate valid and expired CA fixtures @@ -2572,7 +2572,7 @@ func TestSyncOperatorGroups(t *testing.T) { } func TestIsReplacing(t *testing.T) { - log.SetLevel(log.DebugLevel) + logrus.SetLevel(logrus.DebugLevel) namespace := "ns" type initial struct { @@ -2677,7 +2677,7 @@ func TestIsBeingReplaced(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // configure cluster state - op := &Operator{} + op := &Operator{Operator: &queueinformer.Operator{Log: logrus.New()}} require.Equal(t, tt.expected, op.isBeingReplaced(tt.in, tt.initial.csvs)) }) @@ -2725,7 +2725,7 @@ func TestCheckReplacement(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // configure cluster state - op := &Operator{} + op := &Operator{Operator: &queueinformer.Operator{Log: logrus.New()}} require.Equal(t, tt.expected, op.isBeingReplaced(tt.in, tt.initial.csvs)) }) diff --git a/pkg/controller/operators/olm/requirements.go b/pkg/controller/operators/olm/requirements.go index bc98cbbf89d..c9f8746da65 100644 --- a/pkg/controller/operators/olm/requirements.go +++ b/pkg/controller/operators/olm/requirements.go @@ -3,11 +3,11 @@ package olm import ( "encoding/json" "fmt" + "github.com/sirupsen/logrus" "github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1" olmErrors "github.com/operator-framework/operator-lifecycle-manager/pkg/controller/errors" "github.com/operator-framework/operator-lifecycle-manager/pkg/controller/install" - log "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -28,7 +28,7 @@ func (a *Operator) requirementStatus(strategyDetailsDeployment *install.Strategy // check if CRD exists - this verifies group, version, and kind, so no need for GVK check via discovery crd, err := a.lister.APIExtensionsV1beta1().CustomResourceDefinitionLister().Get(r.Name) if err != nil { - log.Debugf("Setting 'met' to false, %v with err: %v", r.Name, err) + a.Log.Debugf("Setting 'met' to false, %v with err: %v", r.Name, err) status.Status = v1alpha1.RequirementStatusReasonNotPresent met = false statuses = append(statuses, status) @@ -159,7 +159,7 @@ func (a *Operator) permissionStatus(strategyDetailsDeployment *install.StrategyD checkPermissions := func(permissions []install.StrategyDeploymentPermissions, namespace string) error { for _, perm := range permissions { saName := perm.ServiceAccountName - log.Debugf("perm.ServiceAccountName: %s", saName) + a.Log.Debugf("perm.ServiceAccountName: %s", saName) var status v1alpha1.RequirementStatus if stored, ok := statusesSet[saName]; !ok { @@ -239,7 +239,7 @@ func (a *Operator) permissionStatus(strategyDetailsDeployment *install.StrategyD statuses := []v1alpha1.RequirementStatus{} for key, status := range statusesSet { - log.Debugf("appending permission status: %s", key) + a.Log.Debugf("appending permission status: %s", key) statuses = append(statuses, status) } @@ -279,14 +279,14 @@ func (a *Operator) requirementAndPermissionStatus(csv *v1alpha1.ClusterServiceVe statuses := append(reqStatuses, permStatuses...) met := reqMet && permMet if !met { - log.Debugf("reqMet=%#v permMet=%v\n", reqMet, permMet) + a.Log.Debugf("reqMet=%#v permMet=%v\n", reqMet, permMet) } return met, statuses, nil } func (a *Operator) isGVKRegistered(group, version, kind string) error { - logger := log.WithFields(log.Fields{ + logger := a.Log.WithFields(logrus.Fields{ "group": group, "version": version, "kind": kind, diff --git a/pkg/lib/operatorclient/client.go b/pkg/lib/operatorclient/client.go index d06f7f9dd65..7c8173e54b0 100644 --- a/pkg/lib/operatorclient/client.go +++ b/pkg/lib/operatorclient/client.go @@ -1,7 +1,7 @@ package operatorclient import ( - log "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -137,20 +137,20 @@ type Client struct { } // NewClient creates a kubernetes client or bails out on on failures. -func NewClientFromConfig(kubeconfig string) ClientInterface { +func NewClientFromConfig(kubeconfig string, logger *logrus.Logger) ClientInterface { var config *rest.Config var err error if kubeconfig != "" { - log.Infof("Loading kube client config from path %q", kubeconfig) + logger.Infof("Loading kube client config from path %q", kubeconfig) config, err = clientcmd.BuildConfigFromFlags("", kubeconfig) } else { - log.Infof("Using in-cluster kube client config") + logger.Infof("Using in-cluster kube client config") config, err = rest.InClusterConfig() } if err != nil { - log.Fatalf("Cannot load config for REST client: %v", err) + logger.Fatalf("Cannot load config for REST client: %v", err) } return &Client{kubernetes.NewForConfigOrDie(config), apiextensions.NewForConfigOrDie(config), apiregistration.NewForConfigOrDie(config)} diff --git a/pkg/lib/queueinformer/queueinformer.go b/pkg/lib/queueinformer/queueinformer.go index b527c4bd59f..69ce3b9cb2e 100644 --- a/pkg/lib/queueinformer/queueinformer.go +++ b/pkg/lib/queueinformer/queueinformer.go @@ -2,7 +2,7 @@ package queueinformer import ( "github.com/operator-framework/operator-lifecycle-manager/pkg/metrics" - log "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" ) @@ -20,6 +20,7 @@ type QueueInformer struct { resourceEventHandlerFuncs *cache.ResourceEventHandlerFuncs name string metrics.MetricsProvider + log *logrus.Logger } // enqueue adds a key to the queue. If obj is a key already it gets added directly. @@ -44,7 +45,7 @@ func (q *QueueInformer) enqueue(obj interface{}) { func (q *QueueInformer) keyFunc(obj interface{}) (string, bool) { k, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) if err != nil { - log.Infof("creating key failed: %s", err) + q.log.Infof("creating key failed: %s", err) return k, false } @@ -57,7 +58,7 @@ func (q *QueueInformer) defaultAddFunc(obj interface{}) { return } - log.Infof("%s added", key) + q.log.Infof("%s added", key) q.enqueue(key) } @@ -67,7 +68,7 @@ func (q *QueueInformer) defaultDeleteFunc(obj interface{}) { return } - log.Infof("%s deleted", key) + q.log.Infof("%s deleted", key) q.queue.Forget(key) } @@ -77,12 +78,12 @@ func (q *QueueInformer) defaultUpdateFunc(oldObj, newObj interface{}) { return } - log.Infof("%s updated", key) + q.log.Infof("%s updated", key) q.enqueue(key) } // defaultResourceEventhandlerFuncs provides the default implementation for responding to events -// these simply log the event and add the object's key to the queue for later processing +// these simply Log the event and add the object's key to the queue for later processing func (q *QueueInformer) defaultResourceEventHandlerFuncs() *cache.ResourceEventHandlerFuncs { return &cache.ResourceEventHandlerFuncs{ AddFunc: q.defaultAddFunc, @@ -93,23 +94,24 @@ func (q *QueueInformer) defaultResourceEventHandlerFuncs() *cache.ResourceEventH // New creates a set of new queueinformers given a name, a set of informers, and a sync handler to handle the objects // that the operator is managing. Optionally, custom event handler funcs can be passed in (defaults will be provided) -func New(queue workqueue.RateLimitingInterface, informers []cache.SharedIndexInformer, handler SyncHandler, funcs *cache.ResourceEventHandlerFuncs, name string, metrics metrics.MetricsProvider) []*QueueInformer { +func New(queue workqueue.RateLimitingInterface, informers []cache.SharedIndexInformer, handler SyncHandler, funcs *cache.ResourceEventHandlerFuncs, name string, metrics metrics.MetricsProvider, logger *logrus.Logger) []*QueueInformer { queueInformers := []*QueueInformer{} for _, informer := range informers { - queueInformers = append(queueInformers, NewInformer(queue, informer, handler, funcs, name, metrics)) + queueInformers = append(queueInformers, NewInformer(queue, informer, handler, funcs, name, metrics, logger)) } return queueInformers } // NewInformer creates a new queueinformer given a name, an informer, and a sync handler to handle the objects // that the operator is managing. Optionally, custom event handler funcs can be passed in (defaults will be provided) -func NewInformer(queue workqueue.RateLimitingInterface, informer cache.SharedIndexInformer, handler SyncHandler, funcs *cache.ResourceEventHandlerFuncs, name string, metrics metrics.MetricsProvider) *QueueInformer { +func NewInformer(queue workqueue.RateLimitingInterface, informer cache.SharedIndexInformer, handler SyncHandler, funcs *cache.ResourceEventHandlerFuncs, name string, metrics metrics.MetricsProvider, logger *logrus.Logger) *QueueInformer { queueInformer := &QueueInformer{ queue: queue, informer: informer, syncHandler: handler, name: name, MetricsProvider: metrics, + log: logger, } queueInformer.resourceEventHandlerFuncs = queueInformer.defaultResourceEventHandlerFuncs() if funcs != nil { diff --git a/pkg/lib/queueinformer/queueinformer_operator.go b/pkg/lib/queueinformer/queueinformer_operator.go index d1ed758f100..545c36a8d9f 100644 --- a/pkg/lib/queueinformer/queueinformer_operator.go +++ b/pkg/lib/queueinformer/queueinformer_operator.go @@ -3,11 +3,12 @@ package queueinformer import ( "fmt" - "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorclient" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/tools/cache" + + "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorclient" ) // An Operator is a collection of QueueInformers @@ -16,28 +17,31 @@ type Operator struct { queueInformers []*QueueInformer informers []cache.SharedIndexInformer OpClient operatorclient.ClientInterface + Log *logrus.Logger } // NewOperator creates a new Operator configured to manage the cluster defined in kubeconfig. -func NewOperator(kubeconfig string, queueInformers ...*QueueInformer) (*Operator, error) { - opClient := operatorclient.NewClientFromConfig(kubeconfig) +func NewOperator(kubeconfig string, logger *logrus.Logger, queueInformers ...*QueueInformer) (*Operator, error) { + opClient := operatorclient.NewClientFromConfig(kubeconfig, logger) if queueInformers == nil { queueInformers = []*QueueInformer{} } operator := &Operator{ OpClient: opClient, queueInformers: queueInformers, + Log: logger, } return operator, nil } -func NewOperatorFromClient(opClient operatorclient.ClientInterface, queueInformers ...*QueueInformer) (*Operator, error) { +func NewOperatorFromClient(opClient operatorclient.ClientInterface, logger *logrus.Logger, queueInformers ...*QueueInformer) (*Operator, error) { if queueInformers == nil { queueInformers = []*QueueInformer{} } operator := &Operator{ OpClient: opClient, queueInformers: queueInformers, + Log: logger, } return operator, nil } @@ -80,7 +84,7 @@ func (o *Operator) Run(stopc <-chan struct{}) (ready, done chan struct{}) { errChan <- errors.Wrap(err, "communicating with server failed") return } - log.Infof("connection established. cluster-version: %v", v) + o.Log.Infof("connection established. cluster-version: %v", v) errChan <- nil }() @@ -95,15 +99,15 @@ func (o *Operator) Run(stopc <-chan struct{}) (ready, done chan struct{}) { select { case err := <-errChan: if err != nil { - log.Infof("operator not ready: %s", err.Error()) + o.Log.Infof("operator not ready: %s", err.Error()) return } - log.Info("operator ready") + o.Log.Info("operator ready") case <-stopc: return } - log.Info("starting informers...") + o.Log.Info("starting informers...") for _, queueInformer := range o.queueInformers { go queueInformer.informer.Run(stopc) } @@ -112,13 +116,13 @@ func (o *Operator) Run(stopc <-chan struct{}) (ready, done chan struct{}) { go informer.Run(stopc) } - log.Info("waiting for caches to sync...") + o.Log.Info("waiting for caches to sync...") if ok := cache.WaitForCacheSync(stopc, hasSyncedCheckFns...); !ok { - log.Info("failed to wait for caches to sync") + o.Log.Info("failed to wait for caches to sync") return } - log.Info("starting workers...") + o.Log.Info("starting workers...") for _, queueInformer := range o.queueInformers { go o.worker(queueInformer) } @@ -147,20 +151,20 @@ func (o *Operator) processNextWorkItem(loop *QueueInformer) bool { // requeue five times on error if err := o.sync(loop, key.(string)); err != nil && queue.NumRequeues(key.(string)) < 5 { - log.Infof("retrying %s", key) + o.Log.Infof("retrying %s", key) utilruntime.HandleError(errors.Wrap(err, fmt.Sprintf("Sync %q failed", key))) queue.AddRateLimited(key) return true } queue.Forget(key) if err := loop.HandleMetrics(); err != nil { - log.Error(err) + o.Log.Error(err) } return true } func (o *Operator) sync(loop *QueueInformer, key string) error { - logger := log.WithField("queue", loop.name).WithField("key", key) + logger := o.Log.WithField("queue", loop.name).WithField("key", key) logger.Info("getting from queue") obj, exists, err := loop.informer.GetIndexer().GetByKey(key) if err != nil { diff --git a/pkg/package-server/provider/inmem.go b/pkg/package-server/provider/inmem.go index 997a4d324ac..a0d5abe198d 100644 --- a/pkg/package-server/provider/inmem.go +++ b/pkg/package-server/provider/inmem.go @@ -7,7 +7,7 @@ import ( "time" "github.com/ghodss/yaml" - log "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/cache" @@ -67,6 +67,7 @@ func NewInMemoryProvider(informers []cache.SharedIndexInformer, queueOperator *q nil, "catsrc", metrics.NewMetricsNil(), + logrus.New(), ) for _, informer := range queueInformers { prov.RegisterQueueInformer(informer) @@ -78,7 +79,7 @@ func NewInMemoryProvider(informers []cache.SharedIndexInformer, queueOperator *q // parsePackageManifestsFromConfigMap returns a list of PackageManifests from a given ConfigMap func parsePackageManifestsFromConfigMap(cm *corev1.ConfigMap, catsrc *operatorsv1alpha1.CatalogSource) ([]packagev1alpha1.PackageManifest, error) { cmName := cm.GetName() - logger := log.WithFields(log.Fields{ + logger := logrus.WithFields(logrus.Fields{ "Action": "Load ConfigMap", "name": cmName, }) @@ -90,14 +91,14 @@ func parsePackageManifestsFromConfigMap(cm *corev1.ConfigMap, catsrc *operatorsv logger.Debug("ConfigMap contains CSVs") csvListJSON, err := yaml.YAMLToJSON([]byte(csvListYaml)) if err != nil { - log.Debugf("Load ConfigMap -- ERROR %s : error=%s", cmName, err) + logrus.Debugf("Load ConfigMap -- ERROR %s : error=%s", cmName, err) return nil, fmt.Errorf("error loading CSV list yaml from ConfigMap %s: %s", cmName, err) } var parsedCSVList []operatorsv1alpha1.ClusterServiceVersion err = json.Unmarshal([]byte(csvListJSON), &parsedCSVList) if err != nil { - log.Debugf("Load ConfigMap -- ERROR %s : error=%s", cmName, err) + logrus.Debugf("Load ConfigMap -- ERROR %s : error=%s", cmName, err) return nil, fmt.Errorf("error parsing CSV list (json) from ConfigMap %s: %s", cmName, err) } @@ -105,7 +106,7 @@ func parsePackageManifestsFromConfigMap(cm *corev1.ConfigMap, catsrc *operatorsv found = true // TODO: add check for invalid CSV definitions - log.Debugf("found csv %s", csv.GetName()) + logrus.Debugf("found csv %s", csv.GetName()) csvs[csv.GetName()] = csv } } @@ -174,7 +175,7 @@ func parsePackageManifestsFromConfigMap(cm *corev1.ConfigMap, catsrc *operatorsv manifest.ObjectMeta.Labels[k] = v } - log.Debugf("retrieved packagemanifest %s", manifest.GetName()) + logrus.Debugf("retrieved packagemanifest %s", manifest.GetName()) manifests = append(manifests, manifest) } } @@ -191,7 +192,7 @@ func (m *InMemoryProvider) syncCatalogSource(obj interface{}) error { // assert as CatalogSource catsrc, ok := obj.(*operatorsv1alpha1.CatalogSource) if !ok { - log.Debugf("wrong type: %#v", obj) + logrus.Debugf("wrong type: %#v", obj) return fmt.Errorf("casting catalog source failed") } diff --git a/pkg/package-server/server/server.go b/pkg/package-server/server/server.go index 88a159b807b..943af6657a0 100644 --- a/pkg/package-server/server/server.go +++ b/pkg/package-server/server/server.go @@ -173,7 +173,7 @@ func (o *PackageServerOptions) Run(stopCh <-chan struct{}) error { catsrcSharedIndexInformers = append(catsrcSharedIndexInformers, nsInformerFactory.Operators().V1alpha1().CatalogSources().Informer()) } - queueOperator, err := queueinformer.NewOperator(o.Kubeconfig) + queueOperator, err := queueinformer.NewOperator(o.Kubeconfig, log.New()) if err != nil { return err } diff --git a/scripts/build_bare.sh b/scripts/build_bare.sh new file mode 100644 index 00000000000..42e1c07f31c --- /dev/null +++ b/scripts/build_bare.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# Note: run from root +# This is used to start and build services for running e2e tests + +set -e + +if [ -z "$NO_MINIKUBE" ]; then + ps x | grep -q [m]inikube || minikube start --kubernetes-version="v1.11.0" --extra-config=apiserver.v=4 || { echo 'Cannot start minikube.'; exit 1; } + eval $(minikube docker-env) || { echo 'Cannot switch to minikube docker'; exit 1; } + kubectl config use-context minikube + umask 0077 && kubectl config view --minify --flatten --context=minikube > minikube.kubeconfig +fi + +kubectl delete crds --all +kubectl delete namespace e2e || true +kubectl wait --for=delete namespace/e2e || true +kubectl create namespace e2e + +# only used for package server, other operators run locally +docker build -t quay.io/coreos/olm:local . diff --git a/scripts/install_bare.sh b/scripts/install_bare.sh new file mode 100755 index 00000000000..52d083abaae --- /dev/null +++ b/scripts/install_bare.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# Note: run from root dir + +set -e + +if [[ ${#@} < 2 ]]; then + echo "Usage: $0 namespace chart" + echo "* namespace: namespace to install into" + echo "* chart: directory of chart manifests to install" + exit 1 +fi + +namespace=$1 +chart=$2 + +# create OLM resources, minus the running components (they will run locally) +for f in ${chart}/*.yaml +do + if [[ $f == *.configmap.yaml ]] + then + kubectl replace --force -f ${f}; + elif [[ $f == *.deployment.yaml ]] + then + # skip olm and catalog operator deployment + continue + else + kubectl apply -f ${f}; + fi +done + +# wait for package server to be ready +#kubectl rollout status -w deployment.apps/package-server --namespace=${namespace} diff --git a/scripts/run_e2e_bare.sh b/scripts/run_e2e_bare.sh new file mode 100755 index 00000000000..0b57ecfd4fb --- /dev/null +++ b/scripts/run_e2e_bare.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +# Note: run from root +# Individual tests can be run by calling ./test/e2e/run_e2e_local.sh TestName + +set -e + +timestamp=$(date +%s) +namespace="e2e-tests-${timestamp}-$RANDOM" + +function cleanup { + kubectl delete namespace ${namespace} + rm -rf test/e2e/resources +} + +function cleanupAndExit { + exitCode=$? + if [ "$exitCode" -ne "0" ]; then + echo "error running tests. logs written to package.log"; + kubectl -n ${namespace} logs -l app=package-server > package.log + else + cleanup + fi + + exit $exitCode +} + +trap cleanupAndExit SIGINT SIGTERM EXIT + + +kubectl create namespace ${namespace} + +./scripts/package-release.sh 1.0.0-e2e test/e2e/resources test/e2e/e2e-bare-values.yaml +./scripts/install_bare.sh ${namespace} test/e2e/resources + +# run tests +if [ -z "$1" ]; then + test_flags=""; +else + test_flags="-test.run ${1}" +fi + +echo "${test_flags}" +go test -tags=bare -mod=vendor -covermode=count -coverpkg ./pkg/controller/... -test.v -test.timeout 20m ${test_flags} ./test/e2e/... -kubeconfig=${KUBECONFIG:-~/.kube/config} -namespace=${namespace} diff --git a/scripts/run_e2e_local.sh b/scripts/run_e2e_local.sh index 62bee52547f..ecfc81b5ac2 100755 --- a/scripts/run_e2e_local.sh +++ b/scripts/run_e2e_local.sh @@ -12,7 +12,7 @@ tmpdir=`mktemp -d 2>/dev/null || mktemp -d -t 'valuetmpdir'` cp test/e2e/e2e-values.yaml ${tmpdir}/e2e-values.yaml echo "namespace: ${namespace}" >> ${tmpdir}/e2e-values.yaml -echo "watchedNamespaces: ${namespace}" >> ${tmpdir}/e2e-values.yaml +#echo "watchedNamespaces: ${namespace}" >> ${tmpdir}/e2e-values.yaml echo "catalog_namespace: ${namespace}" >> ${tmpdir}/e2e-values.yaml ./scripts/package-release.sh 1.0.0-e2e test/e2e/resources ${tmpdir}/e2e-values.yaml diff --git a/test/e2e/catalog_e2e_test.go b/test/e2e/catalog_e2e_test.go index 6ce494b1211..e0e6effc979 100644 --- a/test/e2e/catalog_e2e_test.go +++ b/test/e2e/catalog_e2e_test.go @@ -1,12 +1,12 @@ +// +build !bare + package e2e import ( "fmt" + "github.com/coreos/go-semver/semver" "testing" - "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorclient" - - "github.com/coreos/go-semver/semver" "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" extv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" @@ -15,6 +15,7 @@ import ( "github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1" "github.com/operator-framework/operator-lifecycle-manager/pkg/controller/registry" + "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorclient" ) func TestCatalogLoadingBetweenRestarts(t *testing.T) { @@ -25,10 +26,10 @@ func TestCatalogLoadingBetweenRestarts(t *testing.T) { stableChannel := "stable" packageStable := packageName + "-stable" manifests := []registry.PackageManifest{ - registry.PackageManifest{ + { PackageName: packageName, Channels: []registry.PackageChannel{ - registry.PackageChannel{Name: stableChannel, CurrentCSVName: packageStable}, + {Name: stableChannel, CurrentCSVName: packageStable}, }, DefaultChannelName: stableChannel, }, diff --git a/test/e2e/csv_e2e_test.go b/test/e2e/csv_e2e_test.go index 5f49cde087f..c95f05b2018 100644 --- a/test/e2e/csv_e2e_test.go +++ b/test/e2e/csv_e2e_test.go @@ -471,7 +471,7 @@ func TestCreateCSVRequirementsMetCRD(t *testing.T) { sa.SetName(genName("sa-")) sa.SetNamespace(testNamespace) _, err := c.CreateServiceAccount(&sa) - require.NoError(t, err, "could not create ServiceAccount") + require.NoError(t, err, "could not create ServiceAccount %#v", sa) permissions := []install.StrategyDeploymentPermissions{ { diff --git a/test/e2e/e2e-bare-values.yaml b/test/e2e/e2e-bare-values.yaml new file mode 100644 index 00000000000..efa32dd784f --- /dev/null +++ b/test/e2e/e2e-bare-values.yaml @@ -0,0 +1,38 @@ +rbacApiVersion: rbac.authorization.k8s.io +namespace: operator-lifecycle-manager +catalog_namespace: operator-lifecycle-manager +olm: + replicaCount: 1 + image: + ref: quay.io/coreos/olm:local + pullPolicy: IfNotPresent + service: + internalPort: 8080 + commandArgs: -test.coverprofile=/tmp/coverage/alm-coverage.cov + +catalog: + replicaCount: 1 + image: + ref: quay.io/coreos/olm:local + pullPolicy: IfNotPresent + service: + internalPort: 8080 + commandArgs: -test.coverprofile=/tmp/catalog-coverage.cov + +package: + replicaCount: 1 + image: + ref: quay.io/coreos/olm:local + pullPolicy: IfNotPresent + service: + internalPort: 5443 + commandArgs: -test.coverprofile=/tmp/catalog-coverage.cov + +e2e: + image: + ref: quay.io/coreos/olm-e2e:local + +job_name: e2e + +catalog_sources: + - rh-operators diff --git a/test/e2e/metrics_e2e_test.go b/test/e2e/metrics_e2e_test.go index 6a924b35084..0257ad73324 100644 --- a/test/e2e/metrics_e2e_test.go +++ b/test/e2e/metrics_e2e_test.go @@ -1,3 +1,5 @@ +// +build !bare + package e2e import ( @@ -14,7 +16,7 @@ func TestMetricsEndpoint(t *testing.T) { c := newKubeClient(t) listOptions := metav1.ListOptions{LabelSelector: "app=olm-operator"} - podList, err := c.KubernetesInterface().CoreV1().Pods(e2eNamespace).List(listOptions) + podList, err := c.KubernetesInterface().CoreV1().Pods(testNamespace).List(listOptions) if err != nil { log.Infof("Error %v\n", err) t.Fatalf("Listing pods failed: %v\n", err) @@ -25,7 +27,7 @@ func TestMetricsEndpoint(t *testing.T) { podName := podList.Items[0].GetName() - rawOutput, err := getMetricsFromPod(t, c, podName, e2eNamespace, 8080) + rawOutput, err := getMetricsFromPod(t, c, podName, testNamespace, 8080) if err != nil { t.Fatalf("Metrics test failed: %v\n", err) } diff --git a/test/e2e/setup_test.go b/test/e2e/setup_test.go new file mode 100644 index 00000000000..0181f18368a --- /dev/null +++ b/test/e2e/setup_test.go @@ -0,0 +1,99 @@ +// +build bare + +package e2e + +import ( + "flag" + "io" + "os" + "strings" + "testing" + "time" + + "github.com/sirupsen/logrus" + + "github.com/operator-framework/operator-lifecycle-manager/pkg/api/client" + "github.com/operator-framework/operator-lifecycle-manager/pkg/controller/install" + "github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/catalog" + "github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/olm" + "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorclient" +) + +var ( + kubeConfigPath = flag.String( + "kubeconfig", "", "path to the kubeconfig file") + + watchedNamespaces = flag.String( + "watchedNamespaces", "", "comma separated list of namespaces for alm operator to watch. "+ + "If not set, or set to the empty string (e.g. `-watchedNamespaces=\"\"`), "+ + "olm operator will watch all namespaces in the cluster.") + + namespace = flag.String( + "namespace", "", "namespace where tests will run") + + testNamespace = "" +) + +func TestMain(m *testing.M) { + if err := flag.Set("logtostderr", "true"); err != nil { + panic(err) + } + flag.Parse() + + testNamespace = *namespace + cleaner = newNamespaceCleaner(testNamespace) + namespaces := strings.Split(*watchedNamespaces, ",") + + olmStopCh := make(chan struct{}, 1) + catalogStopCh := make(chan struct{}, 1) + + // operator dependencies + crClient, err := client.NewClient(*kubeConfigPath) + if err != nil { + logrus.WithError(err).Fatalf("error configuring client") + } + + olmLog, err := os.Create("test/log/e2e-olm.log") + if err != nil { + panic(err) + } + defer olmLog.Close() + olmlogger := logrus.New() + mw := io.MultiWriter(os.Stderr, olmLog) + olmlogger.SetOutput(mw) + olmlogger.SetFormatter(&logrus.TextFormatter{ + ForceColors: true, + DisableTimestamp: true, + }) + olmOpClient := operatorclient.NewClientFromConfig(*kubeConfigPath, olmlogger) + + catLog, err := os.Create("test/log/e2e-catalog.log") + if err != nil { + panic(err) + } + defer catLog.Close() + catlogger := logrus.New() + cmw := io.MultiWriter(os.Stderr, catLog) + catlogger.SetOutput(cmw) + catlogger.SetFormatter(&logrus.TextFormatter{ + ForceColors: true, + DisableTimestamp: true, + }) + + // start operators + olmOperator, err := olm.NewOperator(olmlogger, crClient, olmOpClient, &install.StrategyResolver{}, time.Minute, namespaces) + if err != nil { + logrus.WithError(err).Fatalf("error configuring olm") + } + olmready, _ := olmOperator.Run(olmStopCh) + catalogOperator, err := catalog.NewOperator(*kubeConfigPath, catlogger, time.Minute, *namespace, namespaces...) + if err != nil { + logrus.WithError(err).Fatalf("error configuring catalog") + } + catready, _ := catalogOperator.Run(catalogStopCh) + <-olmready + <-catready + + // run tests + os.Exit(m.Run()) +} diff --git a/test/e2e/util_test.go b/test/e2e/util_test.go index f905467cff4..b6735be958e 100644 --- a/test/e2e/util_test.go +++ b/test/e2e/util_test.go @@ -1,9 +1,8 @@ package e2e import ( - "flag" "fmt" - "os" + "github.com/sirupsen/logrus" "strings" "testing" "time" @@ -42,10 +41,8 @@ const ( ) var ( - cleaner *namespaceCleaner - testNamespace = metav1.NamespaceDefault - genName = names.SimpleNameGenerator.GenerateName - e2eNamespace string + cleaner *namespaceCleaner + genName = names.SimpleNameGenerator.GenerateName persistentCatalogNames = []string{ocsConfigMap} nonPersistentCatalogsFieldSelector = createFieldNotEqualSelector("metadata.name", persistentCatalogNames...) @@ -53,16 +50,6 @@ var ( nonPersistentConfigMapsFieldSelector = createFieldNotEqualSelector("metadata.name", persistentConfigMapNames...) ) -func init() { - e2eNamespace = os.Getenv("NAMESPACE") - if e2eNamespace != "" { - testNamespace = e2eNamespace - } - flag.Set("logtostderr", "true") - flag.Parse() - cleaner = newNamespaceCleaner(testNamespace) -} - type namespaceCleaner struct { namespace string skipCleanupOLM bool @@ -91,32 +78,30 @@ func (c *namespaceCleaner) NotifyTestComplete(t *testing.T, cleanup bool) { // newKubeClient configures a client to talk to the cluster defined by KUBECONFIG func newKubeClient(t *testing.T) operatorclient.ClientInterface { - kubeconfigPath := os.Getenv("KUBECONFIG") - if kubeconfigPath == "" { + if kubeConfigPath == nil { t.Log("using in-cluster config") } // TODO: impersonate ALM serviceaccount - return operatorclient.NewClientFromConfig(kubeconfigPath) + // TODO: thread logger from test + return operatorclient.NewClientFromConfig(*kubeConfigPath, logrus.New()) } func newCRClient(t *testing.T) versioned.Interface { - kubeconfigPath := os.Getenv("KUBECONFIG") - if kubeconfigPath == "" { + if kubeConfigPath == nil { t.Log("using in-cluster config") } // TODO: impersonate ALM serviceaccount - crclient, err := client.NewClient(kubeconfigPath) + crclient, err := client.NewClient(*kubeConfigPath) require.NoError(t, err) return crclient } func newPMClient(t *testing.T) pmversioned.Interface { - kubeconfigPath := os.Getenv("KUBECONFIG") - if kubeconfigPath == "" { + if kubeConfigPath == nil { t.Log("using in-cluster config") } // TODO: impersonate ALM serviceaccount - pmc, err := pmclient.NewClient(kubeconfigPath) + pmc, err := pmclient.NewClient(*kubeConfigPath) require.NoError(t, err) return pmc } @@ -289,7 +274,7 @@ func createFieldNotEqualSelector(field string, names ...string) string { func cleanupOLM(t *testing.T, namespace string) { var immediate int64 = 0 crc := newCRClient(t) - c := newKubeClient(t) + //c := newKubeClient(t) // Cleanup non persistent OLM CRs t.Log("cleaning up any remaining non persistent resources...") @@ -300,8 +285,9 @@ func cleanupOLM(t *testing.T, namespace string) { require.NoError(t, crc.OperatorsV1alpha1().Subscriptions(namespace).DeleteCollection(deleteOptions, listOptions)) require.NoError(t, crc.OperatorsV1alpha1().CatalogSources(namespace).DeleteCollection(deleteOptions, metav1.ListOptions{FieldSelector: nonPersistentCatalogsFieldSelector})) + // error: the server does not allow this method on the requested resource // Cleanup non persistent configmaps - require.NoError(t, c.KubernetesInterface().CoreV1().ConfigMaps(namespace).DeleteCollection(deleteOptions, metav1.ListOptions{FieldSelector: nonPersistentConfigMapsFieldSelector})) + //require.NoError(t, c.KubernetesInterface().CoreV1().ConfigMaps(namespace).DeleteCollection(deleteOptions, metav1.ListOptions{FieldSelector: nonPersistentConfigMapsFieldSelector})) } func buildCatalogSourceCleanupFunc(t *testing.T, crc versioned.Interface, namespace string, catalogSource *v1alpha1.CatalogSource) cleanupFunc { diff --git a/test/log/.gitkeep b/test/log/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/vendor/modules.txt b/vendor/modules.txt index 6c5c94b030e..0bda7d7032f 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -304,11 +304,11 @@ k8s.io/apimachinery/pkg/util/validation/field k8s.io/apimachinery/pkg/apis/meta/internalversion k8s.io/apimachinery/pkg/fields k8s.io/apimachinery/pkg/version +k8s.io/apimachinery/pkg/api/resource k8s.io/apimachinery/pkg/util/errors k8s.io/apimachinery/pkg/util/sets k8s.io/apimachinery/pkg/util/validation k8s.io/apimachinery/pkg/util/waitgroup -k8s.io/apimachinery/pkg/api/resource k8s.io/apimachinery/pkg/selection k8s.io/apimachinery/pkg/conversion/queryparams k8s.io/apimachinery/pkg/util/json