From 6aa3fa92bef9dfdd536355331b82376a48ad79e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20R=C3=BCger?= Date: Thu, 8 Apr 2021 16:30:24 +0200 Subject: [PATCH] Replace github.com/golang/glog with k8s.io/klog/v2 glog is effectively unmaintained and the kubernetes ecosystem is mainly using its fork klog Fixes: #1051 --- cmd/kube-router/kube-router.go | 3 + go.mod | 2 +- go.sum | 1 - pkg/cmd/kube-router.go | 8 +- pkg/controllers/netpol/namespace.go | 10 +- .../netpol/network_policy_controller.go | 100 +++---- pkg/controllers/netpol/pod.go | 14 +- pkg/controllers/netpol/policy.go | 14 +- .../proxy/network_service_graceful.go | 18 +- .../proxy/network_services_controller.go | 268 +++++++++--------- .../proxy/service_endpoints_sync.go | 102 +++---- pkg/controllers/routing/aws.go | 12 +- pkg/controllers/routing/bgp_peers.go | 32 +-- pkg/controllers/routing/bgp_policies.go | 14 +- pkg/controllers/routing/ecmp_vip.go | 46 +-- .../routing/network_routes_controller.go | 176 ++++++------ pkg/controllers/routing/pod_egress.go | 8 +- pkg/cri/remote_runtime.go | 6 +- pkg/healthcheck/health_controller.go | 28 +- pkg/metrics/metrics_controller.go | 12 +- pkg/version/version.go | 4 +- 21 files changed, 440 insertions(+), 438 deletions(-) diff --git a/cmd/kube-router/kube-router.go b/cmd/kube-router/kube-router.go index 8a0cb05594..8748df369c 100644 --- a/cmd/kube-router/kube-router.go +++ b/cmd/kube-router/kube-router.go @@ -12,6 +12,7 @@ import ( "github.com/cloudnativelabs/kube-router/pkg/options" "github.com/cloudnativelabs/kube-router/pkg/version" "github.com/spf13/pflag" + "k8s.io/klog/v2" ) func main() { @@ -23,6 +24,8 @@ func main() { } func Main() error { + klog.InitFlags(nil) + config := options.NewKubeRouterConfig() config.AddFlags(pflag.CommandLine) pflag.Parse() diff --git a/go.mod b/go.mod index 2a851950d8..123c5356ba 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,6 @@ require ( github.com/docker/docker v20.10.5+incompatible github.com/docker/go-connections v0.3.0 // indirect github.com/docker/go-units v0.3.3 // indirect - github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/golang/protobuf v1.4.3 github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce // indirect github.com/k-sone/critbitgo v1.4.0 // indirect @@ -38,6 +37,7 @@ require ( k8s.io/apimachinery v0.20.5 k8s.io/client-go v0.20.5 k8s.io/cri-api v0.20.5 + k8s.io/klog/v2 v2.4.0 ) go 1.13 diff --git a/go.sum b/go.sum index 4ce94d9892..3d68704d48 100644 --- a/go.sum +++ b/go.sum @@ -179,7 +179,6 @@ github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= diff --git a/pkg/cmd/kube-router.go b/pkg/cmd/kube-router.go index bb9691e82d..fb73c7c67b 100644 --- a/pkg/cmd/kube-router.go +++ b/pkg/cmd/kube-router.go @@ -14,7 +14,7 @@ import ( "github.com/cloudnativelabs/kube-router/pkg/metrics" "github.com/cloudnativelabs/kube-router/pkg/options" "github.com/cloudnativelabs/kube-router/pkg/version" - "github.com/golang/glog" + "k8s.io/klog/v2" "time" @@ -78,7 +78,7 @@ func (kr *KubeRouter) Run() error { stopCh := make(chan struct{}) if !(kr.Config.RunFirewall || kr.Config.RunServiceProxy || kr.Config.RunRouter) { - glog.Info("Router, Firewall or Service proxy functionality must be specified. Exiting!") + klog.Info("Router, Firewall or Service proxy functionality must be specified. Exiting!") os.Exit(0) } @@ -117,7 +117,7 @@ func (kr *KubeRouter) Run() error { go mc.Run(healthChan, stopCh, &wg) } else if kr.Config.MetricsPort > 65535 { - glog.Errorf("Metrics port must be over 0 and under 65535, given port: %d", kr.Config.MetricsPort) + klog.Errorf("Metrics port must be over 0 and under 65535, given port: %d", kr.Config.MetricsPort) kr.Config.MetricsEnabled = false } else { kr.Config.MetricsEnabled = false @@ -201,7 +201,7 @@ func (kr *KubeRouter) Run() error { signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM) <-ch - glog.Infof("Shutting down the controllers") + klog.Infof("Shutting down the controllers") close(stopCh) wg.Wait() diff --git a/pkg/controllers/netpol/namespace.go b/pkg/controllers/netpol/namespace.go index 4c342092f2..1818bfa10c 100644 --- a/pkg/controllers/netpol/namespace.go +++ b/pkg/controllers/netpol/namespace.go @@ -3,9 +3,9 @@ package netpol import ( "reflect" - "github.com/golang/glog" api "k8s.io/api/core/v1" "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" ) func (npc *NetworkPolicyController) newNamespaceEventHandler() cache.ResourceEventHandler { @@ -27,7 +27,7 @@ func (npc *NetworkPolicyController) newNamespaceEventHandler() cache.ResourceEve return } default: - glog.Errorf("unexpected object type: %v", obj) + klog.Errorf("unexpected object type: %v", obj) } }, } @@ -37,7 +37,7 @@ func (npc *NetworkPolicyController) handleNamespaceAdd(obj *api.Namespace) { if obj.Labels == nil { return } - glog.V(2).Infof("Received update for namespace: %s", obj.Name) + klog.V(2).Infof("Received update for namespace: %s", obj.Name) npc.RequestFullSync() } @@ -46,7 +46,7 @@ func (npc *NetworkPolicyController) handleNamespaceUpdate(oldObj, newObj *api.Na if reflect.DeepEqual(oldObj.Labels, newObj.Labels) { return } - glog.V(2).Infof("Received update for namespace: %s", newObj.Name) + klog.V(2).Infof("Received update for namespace: %s", newObj.Name) npc.RequestFullSync() } @@ -55,7 +55,7 @@ func (npc *NetworkPolicyController) handleNamespaceDelete(obj *api.Namespace) { if obj.Labels == nil { return } - glog.V(2).Infof("Received namespace: %s delete event", obj.Name) + klog.V(2).Infof("Received namespace: %s delete event", obj.Name) npc.RequestFullSync() } diff --git a/pkg/controllers/netpol/network_policy_controller.go b/pkg/controllers/netpol/network_policy_controller.go index 9202f5cc1d..8557c23a32 100644 --- a/pkg/controllers/netpol/network_policy_controller.go +++ b/pkg/controllers/netpol/network_policy_controller.go @@ -17,8 +17,8 @@ import ( "github.com/cloudnativelabs/kube-router/pkg/options" "github.com/cloudnativelabs/kube-router/pkg/utils" "github.com/coreos/go-iptables/iptables" - "github.com/golang/glog" "github.com/prometheus/client_golang/prometheus" + "k8s.io/klog/v2" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" @@ -139,7 +139,7 @@ func (npc *NetworkPolicyController) Run(healthChan chan<- *healthcheck.Controlle defer t.Stop() defer wg.Done() - glog.Info("Starting network policy controller") + klog.Info("Starting network policy controller") npc.healthChan = healthChan // setup kube-router specific top level cutoms chains @@ -147,7 +147,7 @@ func (npc *NetworkPolicyController) Run(healthChan chan<- *healthcheck.Controlle // Full syncs of the network policy controller take a lot of time and can only be processed one at a time, // therefore, we start it in it's own goroutine and request a sync through a single item channel - glog.Info("Starting network policy controller full sync goroutine") + klog.Info("Starting network policy controller full sync goroutine") wg.Add(1) go func(fullSyncRequest <-chan struct{}, stopCh <-chan struct{}, wg *sync.WaitGroup) { defer wg.Done() @@ -155,16 +155,16 @@ func (npc *NetworkPolicyController) Run(healthChan chan<- *healthcheck.Controlle // Add an additional non-blocking select to ensure that if the stopCh channel is closed it is handled first select { case <-stopCh: - glog.Info("Shutting down network policies full sync goroutine") + klog.Info("Shutting down network policies full sync goroutine") return default: } select { case <-stopCh: - glog.Info("Shutting down network policies full sync goroutine") + klog.Info("Shutting down network policies full sync goroutine") return case <-fullSyncRequest: - glog.V(3).Info("Received request for a full sync, processing") + klog.V(3).Info("Received request for a full sync, processing") npc.fullPolicySync() // fullPolicySync() is a blocking request here } } @@ -172,11 +172,11 @@ func (npc *NetworkPolicyController) Run(healthChan chan<- *healthcheck.Controlle // loop forever till notified to stop on stopCh for { - glog.V(1).Info("Requesting periodic sync of iptables to reflect network policies") + klog.V(1).Info("Requesting periodic sync of iptables to reflect network policies") npc.RequestFullSync() select { case <-stopCh: - glog.Infof("Shutting down network policies controller") + klog.Infof("Shutting down network policies controller") return case <-t.C: } @@ -187,9 +187,9 @@ func (npc *NetworkPolicyController) Run(healthChan chan<- *healthcheck.Controlle func (npc *NetworkPolicyController) RequestFullSync() { select { case npc.fullSyncRequestChan <- struct{}{}: - glog.V(3).Info("Full sync request queue was empty so a full sync request was successfully sent") + klog.V(3).Info("Full sync request queue was empty so a full sync request was successfully sent") default: // Don't block if the buffered channel is full, return quickly so that we don't block callee execution - glog.V(1).Info("Full sync request queue was full, skipping...") + klog.V(1).Info("Full sync request queue was full, skipping...") } } @@ -211,52 +211,52 @@ func (npc *NetworkPolicyController) fullPolicySync() { metrics.ControllerIptablesSyncTotalTime.Add(endTime.Seconds()) metrics.ControllerIptablesSyncTotalCount.Add(1) } - glog.V(1).Infof("sync iptables took %v", endTime) + klog.V(1).Infof("sync iptables took %v", endTime) }() - glog.V(1).Infof("Starting sync of iptables with version: %s", syncVersion) + klog.V(1).Infof("Starting sync of iptables with version: %s", syncVersion) // ensure kube-router specific top level chains and corresponding rules exist npc.ensureTopLevelChains() networkPoliciesInfo, err = npc.buildNetworkPoliciesInfo() if err != nil { - glog.Errorf("Aborting sync. Failed to build network policies: %v", err.Error()) + klog.Errorf("Aborting sync. Failed to build network policies: %v", err.Error()) return } npc.filterTableRules.Reset() if err := utils.SaveInto("filter", &npc.filterTableRules); err != nil { - glog.Errorf("Aborting sync. Failed to run iptables-save: %v" + err.Error()) + klog.Errorf("Aborting sync. Failed to run iptables-save: %v" + err.Error()) return } activePolicyChains, activePolicyIPSets, err := npc.syncNetworkPolicyChains(networkPoliciesInfo, syncVersion) if err != nil { - glog.Errorf("Aborting sync. Failed to sync network policy chains: %v" + err.Error()) + klog.Errorf("Aborting sync. Failed to sync network policy chains: %v" + err.Error()) return } activePodFwChains, err := npc.syncPodFirewallChains(networkPoliciesInfo, syncVersion) if err != nil { - glog.Errorf("Aborting sync. Failed to sync pod firewalls: %v", err.Error()) + klog.Errorf("Aborting sync. Failed to sync pod firewalls: %v", err.Error()) return } err = npc.cleanupStaleRules(activePolicyChains, activePodFwChains, activePolicyIPSets) if err != nil { - glog.Errorf("Aborting sync. Failed to cleanup stale iptables rules: %v", err.Error()) + klog.Errorf("Aborting sync. Failed to cleanup stale iptables rules: %v", err.Error()) return } if err := utils.Restore("filter", npc.filterTableRules.Bytes()); err != nil { - glog.Errorf("Aborting sync. Failed to run iptables-restore: %v" + err.Error()) + klog.Errorf("Aborting sync. Failed to run iptables-restore: %v" + err.Error()) return } err = npc.cleanupStaleIPSets(activePolicyIPSets) if err != nil { - glog.Errorf("Failed to cleanup stale ipsets: %v", err.Error()) + klog.Errorf("Failed to cleanup stale ipsets: %v", err.Error()) return } } @@ -270,7 +270,7 @@ func (npc *NetworkPolicyController) ensureTopLevelChains() { iptablesCmdHandler, err := iptables.New() if err != nil { - glog.Fatalf("Failed to initialize iptables executor due to %s", err.Error()) + klog.Fatalf("Failed to initialize iptables executor due to %s", err.Error()) } addUUIDForRuleSpec := func(chain string, ruleSpec *[]string) (string, error) { @@ -288,18 +288,18 @@ func (npc *NetworkPolicyController) ensureTopLevelChains() { ensureRuleAtPosition := func(chain string, ruleSpec []string, uuid string, position int) { exists, err := iptablesCmdHandler.Exists("filter", chain, ruleSpec...) if err != nil { - glog.Fatalf("Failed to verify rule exists in %s chain due to %s", chain, err.Error()) + klog.Fatalf("Failed to verify rule exists in %s chain due to %s", chain, err.Error()) } if !exists { err := iptablesCmdHandler.Insert("filter", chain, position, ruleSpec...) if err != nil { - glog.Fatalf("Failed to run iptables command to insert in %s chain %s", chain, err.Error()) + klog.Fatalf("Failed to run iptables command to insert in %s chain %s", chain, err.Error()) } return } rules, err := iptablesCmdHandler.List("filter", chain) if err != nil { - glog.Fatalf("failed to list rules in filter table %s chain due to %s", chain, err.Error()) + klog.Fatalf("failed to list rules in filter table %s chain due to %s", chain, err.Error()) } var ruleNo, ruleIndexOffset int @@ -320,11 +320,11 @@ func (npc *NetworkPolicyController) ensureTopLevelChains() { if ruleNo != position { err = iptablesCmdHandler.Insert("filter", chain, position, ruleSpec...) if err != nil { - glog.Fatalf("Failed to run iptables command to insert in %s chain %s", chain, err.Error()) + klog.Fatalf("Failed to run iptables command to insert in %s chain %s", chain, err.Error()) } err = iptablesCmdHandler.Delete("filter", chain, strconv.Itoa(ruleNo+1)) if err != nil { - glog.Fatalf("Failed to delete incorrect rule in %s chain due to %s", chain, err.Error()) + klog.Fatalf("Failed to delete incorrect rule in %s chain due to %s", chain, err.Error()) } } } @@ -334,12 +334,12 @@ func (npc *NetworkPolicyController) ensureTopLevelChains() { for builtinChain, customChain := range chains { err = iptablesCmdHandler.NewChain("filter", customChain) if err != nil && err.(*iptables.Error).ExitStatus() != 1 { - glog.Fatalf("Failed to run iptables command to create %s chain due to %s", customChain, err.Error()) + klog.Fatalf("Failed to run iptables command to create %s chain due to %s", customChain, err.Error()) } args := []string{"-m", "comment", "--comment", "kube-router netpol", "-j", customChain} uuid, err := addUUIDForRuleSpec(builtinChain, &args) if err != nil { - glog.Fatalf("Failed to get uuid for rule: %s", err.Error()) + klog.Fatalf("Failed to get uuid for rule: %s", err.Error()) } ensureRuleAtPosition(builtinChain, args, uuid, 1) } @@ -347,7 +347,7 @@ func (npc *NetworkPolicyController) ensureTopLevelChains() { whitelistServiceVips := []string{"-m", "comment", "--comment", "allow traffic to cluster IP", "-d", npc.serviceClusterIPRange.String(), "-j", "RETURN"} uuid, err := addUUIDForRuleSpec(kubeInputChainName, &whitelistServiceVips) if err != nil { - glog.Fatalf("Failed to get uuid for rule: %s", err.Error()) + klog.Fatalf("Failed to get uuid for rule: %s", err.Error()) } ensureRuleAtPosition(kubeInputChainName, whitelistServiceVips, uuid, 1) @@ -355,7 +355,7 @@ func (npc *NetworkPolicyController) ensureTopLevelChains() { "-m", "multiport", "--dports", npc.serviceNodePortRange, "-j", "RETURN"} uuid, err = addUUIDForRuleSpec(kubeInputChainName, &whitelistTCPNodeports) if err != nil { - glog.Fatalf("Failed to get uuid for rule: %s", err.Error()) + klog.Fatalf("Failed to get uuid for rule: %s", err.Error()) } ensureRuleAtPosition(kubeInputChainName, whitelistTCPNodeports, uuid, 2) @@ -363,7 +363,7 @@ func (npc *NetworkPolicyController) ensureTopLevelChains() { "-m", "multiport", "--dports", npc.serviceNodePortRange, "-j", "RETURN"} uuid, err = addUUIDForRuleSpec(kubeInputChainName, &whitelistUDPNodeports) if err != nil { - glog.Fatalf("Failed to get uuid for rule: %s", err.Error()) + klog.Fatalf("Failed to get uuid for rule: %s", err.Error()) } ensureRuleAtPosition(kubeInputChainName, whitelistUDPNodeports, uuid, 3) @@ -371,7 +371,7 @@ func (npc *NetworkPolicyController) ensureTopLevelChains() { whitelistServiceVips := []string{"-m", "comment", "--comment", "allow traffic to external IP range: " + externalIPRange.String(), "-d", externalIPRange.String(), "-j", "RETURN"} uuid, err = addUUIDForRuleSpec(kubeInputChainName, &whitelistServiceVips) if err != nil { - glog.Fatalf("Failed to get uuid for rule: %s", err.Error()) + klog.Fatalf("Failed to get uuid for rule: %s", err.Error()) } ensureRuleAtPosition(kubeInputChainName, whitelistServiceVips, uuid, externalIPIndex+4) } @@ -386,7 +386,7 @@ func (npc *NetworkPolicyController) cleanupStaleRules(activePolicyChains, active // initialize tool sets for working with iptables and ipset iptablesCmdHandler, err := iptables.New() if err != nil { - glog.Fatalf("failed to initialize iptables command executor due to %s", err.Error()) + klog.Fatalf("failed to initialize iptables command executor due to %s", err.Error()) } // find iptables chains and ipsets that are no longer used by comparing current to the active maps we were passed @@ -452,11 +452,11 @@ func (npc *NetworkPolicyController) cleanupStaleIPSets(activePolicyIPSets map[st cleanupPolicyIPSets := make([]*utils.Set, 0) ipsets, err := utils.NewIPSet(false) if err != nil { - glog.Fatalf("failed to create ipsets command executor due to %s", err.Error()) + klog.Fatalf("failed to create ipsets command executor due to %s", err.Error()) } err = ipsets.Save() if err != nil { - glog.Fatalf("failed to initialize ipsets command executor due to %s", err.Error()) + klog.Fatalf("failed to initialize ipsets command executor due to %s", err.Error()) } for _, set := range ipsets.Sets { if strings.HasPrefix(set.Name, kubeSourceIPSetPrefix) || @@ -479,17 +479,17 @@ func (npc *NetworkPolicyController) cleanupStaleIPSets(activePolicyIPSets map[st // Cleanup cleanup configurations done func (npc *NetworkPolicyController) Cleanup() { - glog.Info("Cleaning up iptables configuration permanently done by kube-router") + klog.Info("Cleaning up iptables configuration permanently done by kube-router") iptablesCmdHandler, err := iptables.New() if err != nil { - glog.Errorf("Failed to initialize iptables executor: %s", err.Error()) + klog.Errorf("Failed to initialize iptables executor: %s", err.Error()) } // delete jump rules in FORWARD chain to pod specific firewall chain forwardChainRules, err := iptablesCmdHandler.List("filter", kubeForwardChainName) if err != nil { - glog.Errorf("Failed to delete iptables rules as part of cleanup") + klog.Errorf("Failed to delete iptables rules as part of cleanup") return } @@ -499,7 +499,7 @@ func (npc *NetworkPolicyController) Cleanup() { if strings.Contains(rule, kubePodFirewallChainPrefix) { err = iptablesCmdHandler.Delete("filter", kubeForwardChainName, strconv.Itoa(i-realRuleNo)) if err != nil { - glog.Errorf("Failed to delete iptables rule as part of cleanup: %s", err) + klog.Errorf("Failed to delete iptables rule as part of cleanup: %s", err) } realRuleNo++ } @@ -508,7 +508,7 @@ func (npc *NetworkPolicyController) Cleanup() { // delete jump rules in OUTPUT chain to pod specific firewall chain forwardChainRules, err = iptablesCmdHandler.List("filter", kubeOutputChainName) if err != nil { - glog.Errorf("Failed to delete iptables rules as part of cleanup") + klog.Errorf("Failed to delete iptables rules as part of cleanup") return } @@ -518,7 +518,7 @@ func (npc *NetworkPolicyController) Cleanup() { if strings.Contains(rule, kubePodFirewallChainPrefix) { err = iptablesCmdHandler.Delete("filter", kubeOutputChainName, strconv.Itoa(i-realRuleNo)) if err != nil { - glog.Errorf("Failed to delete iptables rule as part of cleanup: %s", err) + klog.Errorf("Failed to delete iptables rule as part of cleanup: %s", err) } realRuleNo++ } @@ -527,19 +527,19 @@ func (npc *NetworkPolicyController) Cleanup() { // flush and delete pod specific firewall chain chains, err := iptablesCmdHandler.ListChains("filter") if err != nil { - glog.Errorf("Unable to list chains: %s", err) + klog.Errorf("Unable to list chains: %s", err) return } for _, chain := range chains { if strings.HasPrefix(chain, kubePodFirewallChainPrefix) { err = iptablesCmdHandler.ClearChain("filter", chain) if err != nil { - glog.Errorf("Failed to cleanup iptables rules: " + err.Error()) + klog.Errorf("Failed to cleanup iptables rules: " + err.Error()) return } err = iptablesCmdHandler.DeleteChain("filter", chain) if err != nil { - glog.Errorf("Failed to cleanup iptables rules: " + err.Error()) + klog.Errorf("Failed to cleanup iptables rules: " + err.Error()) return } } @@ -548,19 +548,19 @@ func (npc *NetworkPolicyController) Cleanup() { // flush and delete per network policy specific chain chains, err = iptablesCmdHandler.ListChains("filter") if err != nil { - glog.Errorf("Unable to list chains: %s", err) + klog.Errorf("Unable to list chains: %s", err) return } for _, chain := range chains { if strings.HasPrefix(chain, kubeNetworkPolicyChainPrefix) { err = iptablesCmdHandler.ClearChain("filter", chain) if err != nil { - glog.Errorf("Failed to cleanup iptables rules: " + err.Error()) + klog.Errorf("Failed to cleanup iptables rules: " + err.Error()) return } err = iptablesCmdHandler.DeleteChain("filter", chain) if err != nil { - glog.Errorf("Failed to cleanup iptables rules: " + err.Error()) + klog.Errorf("Failed to cleanup iptables rules: " + err.Error()) return } } @@ -569,17 +569,17 @@ func (npc *NetworkPolicyController) Cleanup() { // delete all ipsets ipset, err := utils.NewIPSet(false) if err != nil { - glog.Errorf("Failed to clean up ipsets: " + err.Error()) + klog.Errorf("Failed to clean up ipsets: " + err.Error()) } err = ipset.Save() if err != nil { - glog.Errorf("Failed to clean up ipsets: " + err.Error()) + klog.Errorf("Failed to clean up ipsets: " + err.Error()) } err = ipset.DestroyAllWithin() if err != nil { - glog.Errorf("Failed to clean up ipsets: " + err.Error()) + klog.Errorf("Failed to clean up ipsets: " + err.Error()) } - glog.Infof("Successfully cleaned the iptables configuration done by kube-router") + klog.Infof("Successfully cleaned the iptables configuration done by kube-router") } // NewNetworkPolicyController returns new NetworkPolicyController object diff --git a/pkg/controllers/netpol/pod.go b/pkg/controllers/netpol/pod.go index 590e213c35..9b8671cd64 100644 --- a/pkg/controllers/netpol/pod.go +++ b/pkg/controllers/netpol/pod.go @@ -6,9 +6,9 @@ import ( "reflect" "strings" - "github.com/golang/glog" api "k8s.io/api/core/v1" "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" ) func (npc *NetworkPolicyController) newPodEventHandler() cache.ResourceEventHandler { @@ -36,7 +36,7 @@ func (npc *NetworkPolicyController) newPodEventHandler() cache.ResourceEventHand // OnPodUpdate handles updates to pods from the Kubernetes api server func (npc *NetworkPolicyController) OnPodUpdate(obj interface{}) { pod := obj.(*api.Pod) - glog.V(2).Infof("Received update to pod: %s/%s", pod.Namespace, pod.Name) + klog.V(2).Infof("Received update to pod: %s/%s", pod.Namespace, pod.Name) npc.RequestFullSync() } @@ -46,15 +46,15 @@ func (npc *NetworkPolicyController) handlePodDelete(obj interface{}) { if !ok { tombstone, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { - glog.Errorf("unexpected object type: %v", obj) + klog.Errorf("unexpected object type: %v", obj) return } if pod, ok = tombstone.Obj.(*api.Pod); !ok { - glog.Errorf("unexpected object type: %v", obj) + klog.Errorf("unexpected object type: %v", obj) return } } - glog.V(2).Infof("Received pod: %s/%s delete event", pod.Namespace, pod.Name) + klog.V(2).Infof("Received pod: %s/%s delete event", pod.Namespace, pod.Name) npc.RequestFullSync() } @@ -231,7 +231,7 @@ func (npc *NetworkPolicyController) getIngressNetworkPolicyEnabledPods(networkPo } _, ok := policy.targetPods[pod.Status.PodIP] if ok && (policy.policyType == "both" || policy.policyType == "ingress") { - glog.V(2).Infof("Found pod name: " + pod.ObjectMeta.Name + " namespace: " + pod.ObjectMeta.Namespace + " for which network policies need to be applied.") + klog.V(2).Infof("Found pod name: " + pod.ObjectMeta.Name + " namespace: " + pod.ObjectMeta.Namespace + " for which network policies need to be applied.") nodePods[pod.Status.PodIP] = podInfo{ip: pod.Status.PodIP, name: pod.ObjectMeta.Name, namespace: pod.ObjectMeta.Namespace, @@ -260,7 +260,7 @@ func (npc *NetworkPolicyController) getEgressNetworkPolicyEnabledPods(networkPol } _, ok := policy.targetPods[pod.Status.PodIP] if ok && (policy.policyType == "both" || policy.policyType == "egress") { - glog.V(2).Infof("Found pod name: " + pod.ObjectMeta.Name + " namespace: " + pod.ObjectMeta.Namespace + " for which network policies need to be applied.") + klog.V(2).Infof("Found pod name: " + pod.ObjectMeta.Name + " namespace: " + pod.ObjectMeta.Namespace + " for which network policies need to be applied.") nodePods[pod.Status.PodIP] = podInfo{ip: pod.Status.PodIP, name: pod.ObjectMeta.Name, namespace: pod.ObjectMeta.Namespace, diff --git a/pkg/controllers/netpol/policy.go b/pkg/controllers/netpol/policy.go index 93b5d20408..0e579bed4c 100644 --- a/pkg/controllers/netpol/policy.go +++ b/pkg/controllers/netpol/policy.go @@ -11,7 +11,6 @@ import ( "github.com/cloudnativelabs/kube-router/pkg/metrics" "github.com/cloudnativelabs/kube-router/pkg/utils" - "github.com/golang/glog" api "k8s.io/api/core/v1" networking "k8s.io/api/networking/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -19,6 +18,7 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" listers "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" ) func (npc *NetworkPolicyController) newNetworkPolicyEventHandler() cache.ResourceEventHandler { @@ -40,7 +40,7 @@ func (npc *NetworkPolicyController) newNetworkPolicyEventHandler() cache.Resourc // OnNetworkPolicyUpdate handles updates to network policy from the kubernetes api server func (npc *NetworkPolicyController) OnNetworkPolicyUpdate(obj interface{}) { netpol := obj.(*networking.NetworkPolicy) - glog.V(2).Infof("Received update for network policy: %s/%s", netpol.Namespace, netpol.Name) + klog.V(2).Infof("Received update for network policy: %s/%s", netpol.Namespace, netpol.Name) npc.RequestFullSync() } @@ -50,15 +50,15 @@ func (npc *NetworkPolicyController) handleNetworkPolicyDelete(obj interface{}) { if !ok { tombstone, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { - glog.Errorf("unexpected object type: %v", obj) + klog.Errorf("unexpected object type: %v", obj) return } if netpol, ok = tombstone.Obj.(*networking.NetworkPolicy); !ok { - glog.Errorf("unexpected object type: %v", obj) + klog.Errorf("unexpected object type: %v", obj) return } } - glog.V(2).Infof("Received network policy: %s/%s delete event", netpol.Namespace, netpol.Name) + klog.V(2).Infof("Received network policy: %s/%s delete event", netpol.Namespace, netpol.Name) npc.RequestFullSync() } @@ -73,7 +73,7 @@ func (npc *NetworkPolicyController) syncNetworkPolicyChains(networkPoliciesInfo defer func() { endTime := time.Since(start) metrics.ControllerPolicyChainsSyncTime.Observe(endTime.Seconds()) - glog.V(2).Infof("Syncing network policy chains took %v", endTime) + klog.V(2).Infof("Syncing network policy chains took %v", endTime) }() ipset, err := utils.NewIPSet(false) @@ -138,7 +138,7 @@ func (npc *NetworkPolicyController) syncNetworkPolicyChains(networkPoliciesInfo return nil, nil, fmt.Errorf("failed to perform ipset restore: %s", err.Error()) } - glog.V(2).Infof("Iptables chains in the filter table are synchronized with the network policies.") + klog.V(2).Infof("Iptables chains in the filter table are synchronized with the network policies.") return activePolicyChains, activePolicyIPSets, nil } diff --git a/pkg/controllers/proxy/network_service_graceful.go b/pkg/controllers/proxy/network_service_graceful.go index a8b51ada28..23cdd70eed 100644 --- a/pkg/controllers/proxy/network_service_graceful.go +++ b/pkg/controllers/proxy/network_service_graceful.go @@ -9,8 +9,8 @@ import ( "syscall" "time" - "github.com/golang/glog" "github.com/moby/ipvs" + "k8s.io/klog/v2" ) type gracefulQueue struct { @@ -49,7 +49,7 @@ func (nsc *NetworkServicesController) ipvsDeleteDestination(svc *ipvs.Service, d // flush conntrack when Destination for a UDP service changes if svc.Protocol == syscall.IPPROTO_UDP { if err := nsc.flushConntrackUDP(svc); err != nil { - glog.Errorf("Failed to flush conntrack: %s", err.Error()) + klog.Errorf("Failed to flush conntrack: %s", err.Error()) } } return nil @@ -62,7 +62,7 @@ func (nsc *NetworkServicesController) addToGracefulQueue(req *gracefulRequest) { for _, jobQitem := range nsc.gracefulQueue.queue { if jobQitem.ipvsSvc.Address.Equal(req.ipvsSvc.Address) && jobQitem.ipvsSvc.Port == req.ipvsSvc.Port && jobQitem.ipvsSvc.Protocol == req.ipvsSvc.Protocol { if jobQitem.ipvsDst.Address.Equal(req.ipvsDst.Address) && jobQitem.ipvsDst.Port == req.ipvsDst.Port { - glog.V(2).Infof("Endpoint already scheduled for removal %+v %+v %s", *req.ipvsSvc, *req.ipvsDst, req.gracefulTerminationPeriod.String()) + klog.V(2).Infof("Endpoint already scheduled for removal %+v %+v %s", *req.ipvsSvc, *req.ipvsDst, req.gracefulTerminationPeriod.String()) alreadyExists = true break } @@ -72,10 +72,10 @@ func (nsc *NetworkServicesController) addToGracefulQueue(req *gracefulRequest) { // try to get get Termination grace period from the pod, if unsuccesfull use the default timeout podObj, err := nsc.getPodObjectForEndpoint(req.ipvsDst.Address.String()) if err != nil { - glog.V(1).Infof("Failed to find endpoint with ip: %s err: %s", req.ipvsDst.Address.String(), err.Error()) + klog.V(1).Infof("Failed to find endpoint with ip: %s err: %s", req.ipvsDst.Address.String(), err.Error()) req.gracefulTerminationPeriod = nsc.gracefulPeriod } else { - glog.V(1).Infof("Found pod termination grace period %d for pod %s", *podObj.Spec.TerminationGracePeriodSeconds, podObj.Name) + klog.V(1).Infof("Found pod termination grace period %d for pod %s", *podObj.Spec.TerminationGracePeriodSeconds, podObj.Name) req.gracefulTerminationPeriod = time.Duration(float64(*podObj.Spec.TerminationGracePeriodSeconds) * float64(time.Second)) } nsc.gracefulQueue.queue = append(nsc.gracefulQueue.queue, *req) @@ -101,7 +101,7 @@ func (nsc *NetworkServicesController) gracefulDeleteIpvsDestination(req graceful // Get active and inactive connections for the destination aConn, iConn, err := nsc.getIpvsDestinationConnStats(req.ipvsSvc, req.ipvsDst) if err != nil { - glog.V(1).Infof("Could not get connection stats for destination: %s", err.Error()) + klog.V(1).Infof("Could not get connection stats for destination: %s", err.Error()) } else { // Do we have active or inactive connections to this destination // if we don't, proceed and delete the destination ahead of graceful period @@ -117,9 +117,9 @@ func (nsc *NetworkServicesController) gracefulDeleteIpvsDestination(req graceful //Destination has has one or more conditions for deletion if deleteDestination { - glog.V(2).Infof("Deleting IPVS destination: %s", ipvsDestinationString(req.ipvsDst)) + klog.V(2).Infof("Deleting IPVS destination: %s", ipvsDestinationString(req.ipvsDst)) if err := nsc.ln.ipvsDelDestination(req.ipvsSvc, req.ipvsDst); err != nil { - glog.Errorf("Failed to delete IPVS destination: %s, %s", ipvsDestinationString(req.ipvsDst), err.Error()) + klog.Errorf("Failed to delete IPVS destination: %s, %s", ipvsDestinationString(req.ipvsDst), err.Error()) } } return deleteDestination @@ -152,6 +152,6 @@ func (nsc *NetworkServicesController) flushConntrackUDP(svc *ipvs.Service) error return fmt.Errorf("Failed to delete conntrack entry for endpoint: %s:%d due to %s", svc.Address.String(), svc.Port, err.Error()) } } - glog.V(1).Infof("Deleted conntrack entry for endpoint: %s:%d", svc.Address.String(), svc.Port) + klog.V(1).Infof("Deleted conntrack entry for endpoint: %s:%d", svc.Address.String(), svc.Port) return nil } diff --git a/pkg/controllers/proxy/network_services_controller.go b/pkg/controllers/proxy/network_services_controller.go index 7b6c1b50a8..b42cf4b461 100644 --- a/pkg/controllers/proxy/network_services_controller.go +++ b/pkg/controllers/proxy/network_services_controller.go @@ -24,7 +24,6 @@ import ( "github.com/cloudnativelabs/kube-router/pkg/utils" "github.com/coreos/go-iptables/iptables" "github.com/docker/docker/client" - "github.com/golang/glog" "github.com/moby/ipvs" "github.com/prometheus/client_golang/prometheus" "github.com/vishvananda/netlink" @@ -33,6 +32,7 @@ import ( api "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" ) const ( @@ -106,7 +106,7 @@ func (ln *linuxNetworking) ipAddrDel(iface netlink.Link, ip string) error { naddr := &netlink.Addr{IPNet: &net.IPNet{IP: net.ParseIP(ip), Mask: net.IPv4Mask(255, 255, 255, 255)}, Scope: syscall.RT_SCOPE_LINK} err := netlink.AddrDel(iface, naddr) if err != nil && err.Error() != IfaceHasNoAddr { - glog.Errorf("Failed to verify is external ip %s is assocated with dummy interface %s due to %s", + klog.Errorf("Failed to verify is external ip %s is assocated with dummy interface %s due to %s", naddr.IPNet.IP.String(), KubeDummyIf, err.Error()) } // Delete VIP addition to "local" rt table also, fail silently if not found (DSR special case) @@ -114,7 +114,7 @@ func (ln *linuxNetworking) ipAddrDel(iface netlink.Link, ip string) error { out, err := exec.Command("ip", "route", "delete", "local", ip, "dev", KubeDummyIf, "table", "local", "proto", "kernel", "scope", "host", "src", NodeIP.String(), "table", "local").CombinedOutput() if err != nil && !strings.Contains(string(out), "No such process") { - glog.Errorf("Failed to delete route to service VIP %s configured on %s. Error: %v, Output: %s", ip, KubeDummyIf, err, out) + klog.Errorf("Failed to delete route to service VIP %s configured on %s. Error: %v, Output: %s", ip, KubeDummyIf, err, out) } } return err @@ -127,7 +127,7 @@ func (ln *linuxNetworking) ipAddrAdd(iface netlink.Link, ip string, addRoute boo naddr := &netlink.Addr{IPNet: &net.IPNet{IP: net.ParseIP(ip), Mask: net.IPv4Mask(255, 255, 255, 255)}, Scope: syscall.RT_SCOPE_LINK} err := netlink.AddrAdd(iface, naddr) if err != nil && err.Error() != IfaceHasAddr { - glog.Errorf("Failed to assign cluster ip %s to dummy interface: %s", + klog.Errorf("Failed to assign cluster ip %s to dummy interface: %s", naddr.IPNet.IP.String(), err.Error()) return err } @@ -146,7 +146,7 @@ func (ln *linuxNetworking) ipAddrAdd(iface netlink.Link, ip string, addRoute boo out, err := exec.Command("ip", "route", "replace", "local", ip, "dev", KubeDummyIf, "table", "local", "proto", "kernel", "scope", "host", "src", NodeIP.String(), "table", "local").CombinedOutput() if err != nil { - glog.Errorf("Failed to replace route to service VIP %s configured on %s. Error: %v, Output: %s", ip, KubeDummyIf, err, out) + klog.Errorf("Failed to replace route to service VIP %s configured on %s. Error: %v, Output: %s", ip, KubeDummyIf, err, out) } return nil } @@ -291,36 +291,36 @@ func (nsc *NetworkServicesController) Run(healthChan chan<- *healthcheck.Control defer wg.Done() defer close(nsc.syncChan) - glog.Infof("Starting network services controller") + klog.Infof("Starting network services controller") - glog.V(1).Info("Performing cleanup of depreciated masquerade iptables rules (if needed).") + klog.V(1).Info("Performing cleanup of depreciated masquerade iptables rules (if needed).") err := nsc.deleteBadMasqueradeIptablesRules() if err != nil { - glog.Errorf("Error cleaning up old/bad masquerade rules: %s", err.Error()) + klog.Errorf("Error cleaning up old/bad masquerade rules: %s", err.Error()) } // enable masquerade rule err = nsc.ensureMasqueradeIptablesRule() if err != nil { - glog.Errorf("Failed to do add masquerade rule in POSTROUTING chain of nat table due to: %s", err.Error()) + klog.Errorf("Failed to do add masquerade rule in POSTROUTING chain of nat table due to: %s", err.Error()) } // https://www.kernel.org/doc/Documentation/networking/ipvs-sysctl.txt // enable ipvs connection tracking sysctlErr := utils.SetSysctl("net/ipv4/vs/conntrack", 1) if sysctlErr != nil { - glog.Error(sysctlErr.Error()) + klog.Error(sysctlErr.Error()) } // LVS failover not working with UDP packets https://access.redhat.com/solutions/58653 sysctlErr = utils.SetSysctl("net/ipv4/vs/expire_nodest_conn", 1) if sysctlErr != nil { - glog.Error(sysctlErr.Error()) + klog.Error(sysctlErr.Error()) } // LVS failover not working with UDP packets https://access.redhat.com/solutions/58653 sysctlErr = utils.SetSysctl("net/ipv4/vs/expire_quiescent_template", 1) if sysctlErr != nil { - glog.Error(sysctlErr.Error()) + klog.Error(sysctlErr.Error()) } // https://github.com/kubernetes/kubernetes/pull/71114 @@ -329,28 +329,28 @@ func (nsc *NetworkServicesController) Run(healthChan chan<- *healthcheck.Control // Check if the error is fatal, on older kernels this option does not exist and the same behaviour is default // if option is not found just log it if sysctlErr.IsFatal() { - glog.Fatal(sysctlErr.Error()) + klog.Fatal(sysctlErr.Error()) } else { - glog.Info(sysctlErr.Error()) + klog.Info(sysctlErr.Error()) } } // https://github.com/kubernetes/kubernetes/pull/70530/files sysctlErr = utils.SetSysctl("net/ipv4/conf/all/arp_ignore", 1) if sysctlErr != nil { - glog.Error(sysctlErr.Error()) + klog.Error(sysctlErr.Error()) } // https://github.com/kubernetes/kubernetes/pull/70530/files sysctlErr = utils.SetSysctl("net/ipv4/conf/all/arp_announce", 2) if sysctlErr != nil { - glog.Error(sysctlErr.Error()) + klog.Error(sysctlErr.Error()) } // https://github.com/cloudnativelabs/kube-router/issues/282 err = nsc.setupIpvsFirewall() if err != nil { - glog.Error("Error setting up ipvs firewall: " + err.Error()) + klog.Error("Error setting up ipvs firewall: " + err.Error()) } nsc.ProxyFirewallSetup.Broadcast() @@ -359,12 +359,12 @@ func (nsc *NetworkServicesController) Run(healthChan chan<- *healthcheck.Control select { case <-stopCh: - glog.Info("Shutting down network services controller") + klog.Info("Shutting down network services controller") return default: err := nsc.doSync() if err != nil { - glog.Fatalf("Failed to perform initial full sync %s", err.Error()) + klog.Fatalf("Failed to perform initial full sync %s", err.Error()) } nsc.readyForUpdates = true } @@ -376,12 +376,12 @@ func (nsc *NetworkServicesController) Run(healthChan chan<- *healthcheck.Control nsc.mu.Lock() nsc.readyForUpdates = false nsc.mu.Unlock() - glog.Info("Shutting down network services controller") + klog.Info("Shutting down network services controller") return case <-gracefulTicker.C: if nsc.readyForUpdates && nsc.gracefulTermination { - glog.V(3).Info("Performing periodic graceful destination cleanup") + klog.V(3).Info("Performing periodic graceful destination cleanup") nsc.gracefulSync() } @@ -389,18 +389,18 @@ func (nsc *NetworkServicesController) Run(healthChan chan<- *healthcheck.Control healthcheck.SendHeartBeat(healthChan, "NSC") switch perform { case synctypeAll: - glog.V(1).Info("Performing requested full sync of services") + klog.V(1).Info("Performing requested full sync of services") err := nsc.doSync() if err != nil { - glog.Errorf("Error during full sync in network service controller. Error: " + err.Error()) + klog.Errorf("Error during full sync in network service controller. Error: " + err.Error()) } case synctypeIpvs: - glog.V(1).Info("Performing requested sync of ipvs services") + klog.V(1).Info("Performing requested sync of ipvs services") nsc.mu.Lock() err := nsc.syncIpvsServices(nsc.serviceMap, nsc.endpointsMap) nsc.mu.Unlock() if err != nil { - glog.Errorf("Error during ipvs sync in network service controller. Error: " + err.Error()) + klog.Errorf("Error during ipvs sync in network service controller. Error: " + err.Error()) } } if err == nil { @@ -408,12 +408,12 @@ func (nsc *NetworkServicesController) Run(healthChan chan<- *healthcheck.Control } case <-t.C: - glog.V(1).Info("Performing periodic sync of ipvs services") + klog.V(1).Info("Performing periodic sync of ipvs services") healthcheck.SendHeartBeat(healthChan, "NSC") err := nsc.doSync() if err != nil { - glog.Errorf("Error during periodic ipvs sync in network service controller. Error: " + err.Error()) - glog.Errorf("Skipping sending heartbeat from network service controller as periodic sync failed.") + klog.Errorf("Error during periodic ipvs sync in network service controller. Error: " + err.Error()) + klog.Errorf("Skipping sending heartbeat from network service controller as periodic sync failed.") } else { healthcheck.SendHeartBeat(healthChan, "NSC") } @@ -425,7 +425,7 @@ func (nsc *NetworkServicesController) sync(syncType int) { select { case nsc.syncChan <- syncType: default: - glog.V(2).Infof("Already pending sync, dropping request for type %d", syncType) + klog.V(2).Infof("Already pending sync, dropping request for type %d", syncType) } } @@ -437,26 +437,26 @@ func (nsc *NetworkServicesController) doSync() error { // enable masquerade rule err = nsc.ensureMasqueradeIptablesRule() if err != nil { - glog.Errorf("Failed to do add masquerade rule in POSTROUTING chain of nat table due to: %s", err.Error()) + klog.Errorf("Failed to do add masquerade rule in POSTROUTING chain of nat table due to: %s", err.Error()) } nsc.serviceMap = nsc.buildServicesInfo() nsc.endpointsMap = nsc.buildEndpointsInfo() err = nsc.syncHairpinIptablesRules() if err != nil { - glog.Errorf("Error syncing hairpin iptables rules: %s", err.Error()) + klog.Errorf("Error syncing hairpin iptables rules: %s", err.Error()) } err = nsc.syncIpvsServices(nsc.serviceMap, nsc.endpointsMap) if err != nil { - glog.Errorf("Error syncing IPVS services: %s", err.Error()) + klog.Errorf("Error syncing IPVS services: %s", err.Error()) return err } if nsc.MetricsEnabled { err = nsc.publishMetrics(nsc.serviceMap) if err != nil { - glog.Errorf("Error publishing metrics: %v", err) + klog.Errorf("Error publishing metrics: %v", err) return err } } @@ -607,43 +607,43 @@ func (nsc *NetworkServicesController) cleanupIpvsFirewall() { // Clear iptables rules. iptablesCmdHandler, err := iptables.New() if err != nil { - glog.Errorf("Failed to initialize iptables executor: %s", err.Error()) + klog.Errorf("Failed to initialize iptables executor: %s", err.Error()) } else { ipvsFirewallInputChainRule := getIpvsFirewallInputChainRule() err = iptablesCmdHandler.Delete("filter", "INPUT", ipvsFirewallInputChainRule...) if err != nil { - glog.Errorf("Failed to run iptables command: %s", err.Error()) + klog.Errorf("Failed to run iptables command: %s", err.Error()) } err = iptablesCmdHandler.ClearChain("filter", ipvsFirewallChainName) if err != nil { - glog.Errorf("Failed to run iptables command: %s", err.Error()) + klog.Errorf("Failed to run iptables command: %s", err.Error()) } err = iptablesCmdHandler.DeleteChain("filter", ipvsFirewallChainName) if err != nil { - glog.Errorf("Failed to run iptables command: %s", err.Error()) + klog.Errorf("Failed to run iptables command: %s", err.Error()) } } // Clear ipsets. ipSetHandler, err := utils.NewIPSet(false) if err != nil { - glog.Errorf("Failed to initialize ipset handler: %s", err.Error()) + klog.Errorf("Failed to initialize ipset handler: %s", err.Error()) } else { err = ipSetHandler.Destroy(localIPsIPSetName) if err != nil { - glog.Errorf("failed to destroy ipset: %s", err.Error()) + klog.Errorf("failed to destroy ipset: %s", err.Error()) } err = ipSetHandler.Destroy(serviceIPsIPSetName) if err != nil { - glog.Errorf("failed to destroy ipset: %s", err.Error()) + klog.Errorf("failed to destroy ipset: %s", err.Error()) } err = ipSetHandler.Destroy(ipvsServicesIPSetName) if err != nil { - glog.Errorf("failed to destroy ipset: %s", err.Error()) + klog.Errorf("failed to destroy ipset: %s", err.Error()) } } } @@ -693,7 +693,7 @@ func (nsc *NetworkServicesController) syncIpvsFirewall() error { } else if ipvsService.FWMark != 0 { address, protocol, port, err = nsc.lookupServiceByFWMark(ipvsService.FWMark) if err != nil { - glog.Errorf("failed to lookup %d by FWMark: %s", ipvsService.FWMark, err) + klog.Errorf("failed to lookup %d by FWMark: %s", ipvsService.FWMark, err) } if address == "" { continue @@ -727,7 +727,7 @@ func (nsc *NetworkServicesController) publishMetrics(serviceInfoMap serviceInfoM start := time.Now() defer func() { endTime := time.Since(start) - glog.V(2).Infof("Publishing IPVS metrics took %v", endTime) + klog.V(2).Infof("Publishing IPVS metrics took %v", endTime) if nsc.MetricsEnabled { metrics.ControllerIpvsMetricsExportTime.Observe(endTime.Seconds()) } @@ -738,7 +738,7 @@ func (nsc *NetworkServicesController) publishMetrics(serviceInfoMap serviceInfoM return errors.New("Failed to list IPVS services: " + err.Error()) } - glog.V(1).Info("Publishing IPVS metrics") + klog.V(1).Info("Publishing IPVS metrics") for _, svc := range serviceInfoMap { var protocol uint16 var pushMetric bool @@ -775,7 +775,7 @@ func (nsc *NetworkServicesController) publishMetrics(serviceInfoMap serviceInfoM } if pushMetric { - glog.V(3).Infof("Publishing metrics for %s/%s (%s:%d/%s)", svc.namespace, svc.name, svcVip, svc.port, svc.protocol) + klog.V(3).Infof("Publishing metrics for %s/%s (%s:%d/%s)", svc.namespace, svc.name, svcVip, svc.port, svc.protocol) metrics.ServiceBpsIn.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.BPSIn)) metrics.ServiceBpsOut.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.BPSOut)) metrics.ServiceBytesIn.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.BytesIn)) @@ -802,9 +802,9 @@ func (nsc *NetworkServicesController) OnEndpointsUpdate(ep *api.Endpoints) { nsc.mu.Lock() defer nsc.mu.Unlock() - glog.V(1).Infof("Received update to endpoint: %s/%s from watch API", ep.Namespace, ep.Name) + klog.V(1).Infof("Received update to endpoint: %s/%s from watch API", ep.Namespace, ep.Name) if !nsc.readyForUpdates { - glog.V(3).Infof("Skipping update to endpoint: %s/%s as controller is not ready to process service and endpoints updates", ep.Namespace, ep.Name) + klog.V(3).Infof("Skipping update to endpoint: %s/%s as controller is not ready to process service and endpoints updates", ep.Namespace, ep.Name) return } @@ -814,7 +814,7 @@ func (nsc *NetworkServicesController) OnEndpointsUpdate(ep *api.Endpoints) { // ClusterIP before. svc, exists, err := utils.ServiceForEndpoints(&nsc.svcLister, ep) if err != nil { - glog.Errorf("failed to convert endpoints resource to service: %s", err) + klog.Errorf("failed to convert endpoints resource to service: %s", err) return } // ignore updates to Endpoints object with no corresponding Service object @@ -822,7 +822,7 @@ func (nsc *NetworkServicesController) OnEndpointsUpdate(ep *api.Endpoints) { return } if utils.ServiceIsHeadless(svc) { - glog.V(1).Infof("The service associated with endpoint: %s/%s is headless, skipping...", ep.Namespace, ep.Name) + klog.V(1).Infof("The service associated with endpoint: %s/%s is headless, skipping...", ep.Namespace, ep.Name) return } @@ -833,10 +833,10 @@ func (nsc *NetworkServicesController) OnEndpointsUpdate(ep *api.Endpoints) { if len(newEndpointsMap) != len(nsc.endpointsMap) || !reflect.DeepEqual(newEndpointsMap, nsc.endpointsMap) { nsc.endpointsMap = newEndpointsMap nsc.serviceMap = newServiceMap - glog.V(1).Infof("Syncing IPVS services sync for update to endpoint: %s/%s", ep.Namespace, ep.Name) + klog.V(1).Infof("Syncing IPVS services sync for update to endpoint: %s/%s", ep.Namespace, ep.Name) nsc.sync(synctypeIpvs) } else { - glog.V(1).Infof("Skipping IPVS services sync on endpoint: %s/%s update as nothing changed", ep.Namespace, ep.Name) + klog.V(1).Infof("Skipping IPVS services sync on endpoint: %s/%s update as nothing changed", ep.Namespace, ep.Name) } } @@ -846,9 +846,9 @@ func (nsc *NetworkServicesController) OnServiceUpdate(svc *api.Service) { nsc.mu.Lock() defer nsc.mu.Unlock() - glog.V(1).Infof("Received update to service: %s/%s from watch API", svc.Namespace, svc.Name) + klog.V(1).Infof("Received update to service: %s/%s from watch API", svc.Namespace, svc.Name) if !nsc.readyForUpdates { - glog.V(3).Infof("Skipping update to service: %s/%s as controller is not ready to process service and endpoints updates", svc.Namespace, svc.Name) + klog.V(3).Infof("Skipping update to service: %s/%s as controller is not ready to process service and endpoints updates", svc.Namespace, svc.Name) return } @@ -857,7 +857,7 @@ func (nsc *NetworkServicesController) OnServiceUpdate(svc *api.Service) { // need to consider previous versions of the service here as we are guaranteed if is a ClusterIP now, it was a // ClusterIP before. if utils.ServiceIsHeadless(svc) { - glog.V(1).Infof("%s/%s is headless, skipping...", svc.Namespace, svc.Name) + klog.V(1).Infof("%s/%s is headless, skipping...", svc.Namespace, svc.Name) return } @@ -868,10 +868,10 @@ func (nsc *NetworkServicesController) OnServiceUpdate(svc *api.Service) { if len(newServiceMap) != len(nsc.serviceMap) || !reflect.DeepEqual(newServiceMap, nsc.serviceMap) { nsc.endpointsMap = newEndpointsMap nsc.serviceMap = newServiceMap - glog.V(1).Infof("Syncing IPVS services sync on update to service: %s/%s", svc.Namespace, svc.Name) + klog.V(1).Infof("Syncing IPVS services sync on update to service: %s/%s", svc.Namespace, svc.Name) nsc.sync(synctypeIpvs) } else { - glog.V(1).Infof("Skipping syncing IPVS services for update to service: %s/%s as nothing changed", svc.Namespace, svc.Name) + klog.V(1).Infof("Skipping syncing IPVS services for update to service: %s/%s as nothing changed", svc.Namespace, svc.Name) } } @@ -930,7 +930,7 @@ func (ln *linuxNetworking) prepareEndpointForDsr(containerID string, endpointIP if err != nil { return errors.New("Failed to get namespace due to " + err.Error()) } - glog.V(1).Infof("Current network namespace before netns.Set: " + activeNetworkNamespaceHandle.String()) + klog.V(1).Infof("Current network namespace before netns.Set: " + activeNetworkNamespaceHandle.String()) activeNetworkNamespaceHandle.Close() dockerClient, err := client.NewClientWithOpts(client.FromEnv) @@ -960,7 +960,7 @@ func (ln *linuxNetworking) prepareEndpointForDsr(containerID string, endpointIP if err != nil { return errors.New("Failed to get activeNetworkNamespace due to " + err.Error()) } - glog.V(2).Infof("Current network namespace after netns. Set to container network namespace: " + activeNetworkNamespaceHandle.String()) + klog.V(2).Infof("Current network namespace after netns. Set to container network namespace: " + activeNetworkNamespaceHandle.String()) activeNetworkNamespaceHandle.Close() // TODO: fix boilerplate `netns.Set(hostNetworkNamespaceHandle)` code. Need a robust @@ -975,12 +975,12 @@ func (ln *linuxNetworking) prepareEndpointForDsr(containerID string, endpointIP return errors.New("Failed to get hostNetworkNamespace due to " + err.Error()) } activeNetworkNamespaceHandle, err = netns.Get() - glog.V(2).Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) + klog.V(2).Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) activeNetworkNamespaceHandle.Close() return errors.New("Failed to verify if ipip tunnel interface exists in endpoint " + endpointIP + " namespace due to " + err.Error()) } - glog.V(2).Infof("Could not find tunnel interface " + KubeTunnelIf + " in endpoint " + endpointIP + " so creating one.") + klog.V(2).Infof("Could not find tunnel interface " + KubeTunnelIf + " in endpoint " + endpointIP + " so creating one.") ipTunLink := netlink.Iptun{ LinkAttrs: netlink.LinkAttrs{Name: KubeTunnelIf}, Local: net.ParseIP(endpointIP), @@ -992,7 +992,7 @@ func (ln *linuxNetworking) prepareEndpointForDsr(containerID string, endpointIP return errors.New("Failed to set hostNetworkNamespace handle due to " + err.Error()) } activeNetworkNamespaceHandle, err = netns.Get() - glog.V(2).Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) + klog.V(2).Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) activeNetworkNamespaceHandle.Close() return errors.New("Failed to add ipip tunnel interface in endpoint namespace due to " + err.Error()) } @@ -1006,7 +1006,7 @@ func (ln *linuxNetworking) prepareEndpointForDsr(containerID string, endpointIP break } if err != nil && err.Error() == IfaceNotFound { - glog.V(3).Infof("Waiting for tunnel interface %s to come up in the pod, retrying", KubeTunnelIf) + klog.V(3).Infof("Waiting for tunnel interface %s to come up in the pod, retrying", KubeTunnelIf) continue } else { break @@ -1019,12 +1019,12 @@ func (ln *linuxNetworking) prepareEndpointForDsr(containerID string, endpointIP return errors.New("Failed to set hostNetworkNamespace handle due to " + err.Error()) } activeNetworkNamespaceHandle, err = netns.Get() - glog.V(2).Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) + klog.V(2).Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) activeNetworkNamespaceHandle.Close() return errors.New("Failed to get " + KubeTunnelIf + " tunnel interface handle due to " + err.Error()) } - glog.V(2).Infof("Successfully created tunnel interface " + KubeTunnelIf + " in endpoint " + endpointIP + ".") + klog.V(2).Infof("Successfully created tunnel interface " + KubeTunnelIf + " in endpoint " + endpointIP + ".") } // bring the tunnel interface up @@ -1035,7 +1035,7 @@ func (ln *linuxNetworking) prepareEndpointForDsr(containerID string, endpointIP return errors.New("Failed to set hostNetworkNamespace handle due to " + err.Error()) } activeNetworkNamespaceHandle, err = netns.Get() - glog.Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) + klog.Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) activeNetworkNamespaceHandle.Close() return errors.New("Failed to bring up ipip tunnel interface in endpoint namespace due to " + err.Error()) } @@ -1052,11 +1052,11 @@ func (ln *linuxNetworking) prepareEndpointForDsr(containerID string, endpointIP return errors.New("Failed to get activeNetworkNamespace handle due to " + err.Error()) } - glog.V(2).Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) + klog.V(2).Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) activeNetworkNamespaceHandle.Close() return errors.New("Failed to assign vip " + vip + " to kube-tunnel-if interface ") } - glog.Infof("Successfully assinged VIP: " + vip + " in endpoint " + endpointIP + ".") + klog.Infof("Successfully assinged VIP: " + vip + " in endpoint " + endpointIP + ".") // disable rp_filter on all interface err = ioutil.WriteFile("/proc/sys/net/ipv4/conf/kube-tunnel-if/rp_filter", []byte(strconv.Itoa(0)), 0640) @@ -1070,7 +1070,7 @@ func (ln *linuxNetworking) prepareEndpointForDsr(containerID string, endpointIP return errors.New("Failed to get activeNetworkNamespace handle due to " + err.Error()) } - glog.Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) + klog.Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) activeNetworkNamespaceHandle.Close() return errors.New("Failed to disable rp_filter on kube-tunnel-if in the endpoint container") } @@ -1085,7 +1085,7 @@ func (ln *linuxNetworking) prepareEndpointForDsr(containerID string, endpointIP if err != nil { return errors.New("Failed to get activeNetworkNamespace handle due to " + err.Error()) } - glog.Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) + klog.Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) activeNetworkNamespaceHandle.Close() return errors.New("Failed to disable rp_filter on eth0 in the endpoint container") } @@ -1100,12 +1100,12 @@ func (ln *linuxNetworking) prepareEndpointForDsr(containerID string, endpointIP if err != nil { return errors.New("Failed to get activeNetworkNamespace handle due to " + err.Error()) } - glog.V(2).Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) + klog.V(2).Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) activeNetworkNamespaceHandle.Close() return errors.New("Failed to disable rp_filter on `all` in the endpoint container") } - glog.Infof("Successfully disabled rp_filter in endpoint " + endpointIP + ".") + klog.Infof("Successfully disabled rp_filter in endpoint " + endpointIP + ".") err = netns.Set(hostNetworkNamespaceHandle) if err != nil { @@ -1115,7 +1115,7 @@ func (ln *linuxNetworking) prepareEndpointForDsr(containerID string, endpointIP if err != nil { return errors.New("Failed to get activeNetworkNamespace handle due to " + err.Error()) } - glog.Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) + klog.Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) activeNetworkNamespaceHandle.Close() return nil } @@ -1140,7 +1140,7 @@ func (ln *linuxNetworking) prepareEndpointForDsrWithCRI(runtimeEndpoint, contain if err != nil { return errors.New("failed to get host namespace due to " + err.Error()) } - glog.V(1).Infof("current network namespace before netns.Set: " + hostNetworkNamespaceHandle.String()) + klog.V(1).Infof("current network namespace before netns.Set: " + hostNetworkNamespaceHandle.String()) defer hostNetworkNamespaceHandle.Close() rs, err := cri.NewRemoteRuntimeService(runtimeEndpoint, cri.DefaultConnectionTimeout) @@ -1172,7 +1172,7 @@ func (ln *linuxNetworking) prepareEndpointForDsrWithCRI(runtimeEndpoint, contain if err != nil { return errors.New("Failed to get activeNetworkNamespace due to " + err.Error()) } - glog.V(2).Infof("Current network namespace after netns. Set to container network namespace: " + activeNetworkNamespaceHandle.String()) + klog.V(2).Infof("Current network namespace after netns. Set to container network namespace: " + activeNetworkNamespaceHandle.String()) activeNetworkNamespaceHandle.Close() // TODO: fix boilerplate `netns.Set(hostNetworkNamespaceHandle)` code. Need a robust @@ -1187,12 +1187,12 @@ func (ln *linuxNetworking) prepareEndpointForDsrWithCRI(runtimeEndpoint, contain return errors.New("Failed to get hostNetworkNamespace due to " + err.Error()) } activeNetworkNamespaceHandle, err = netns.Get() - glog.V(2).Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) + klog.V(2).Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) activeNetworkNamespaceHandle.Close() return errors.New("Failed to verify if ipip tunnel interface exists in endpoint " + endpointIP + " namespace due to " + err.Error()) } - glog.V(2).Infof("Could not find tunnel interface " + KubeTunnelIf + " in endpoint " + endpointIP + " so creating one.") + klog.V(2).Infof("Could not find tunnel interface " + KubeTunnelIf + " in endpoint " + endpointIP + " so creating one.") ipTunLink := netlink.Iptun{ LinkAttrs: netlink.LinkAttrs{Name: KubeTunnelIf}, Local: net.ParseIP(endpointIP), @@ -1204,7 +1204,7 @@ func (ln *linuxNetworking) prepareEndpointForDsrWithCRI(runtimeEndpoint, contain return errors.New("Failed to set hostNetworkNamespace handle due to " + err.Error()) } activeNetworkNamespaceHandle, err = netns.Get() - glog.V(2).Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) + klog.V(2).Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) activeNetworkNamespaceHandle.Close() return errors.New("Failed to add ipip tunnel interface in endpoint namespace due to " + err.Error()) } @@ -1218,7 +1218,7 @@ func (ln *linuxNetworking) prepareEndpointForDsrWithCRI(runtimeEndpoint, contain break } if err != nil && err.Error() == IfaceNotFound { - glog.V(3).Infof("Waiting for tunnel interface %s to come up in the pod, retrying", KubeTunnelIf) + klog.V(3).Infof("Waiting for tunnel interface %s to come up in the pod, retrying", KubeTunnelIf) continue } else { break @@ -1231,12 +1231,12 @@ func (ln *linuxNetworking) prepareEndpointForDsrWithCRI(runtimeEndpoint, contain return errors.New("Failed to set hostNetworkNamespace handle due to " + err.Error()) } activeNetworkNamespaceHandle, err = netns.Get() - glog.V(2).Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) + klog.V(2).Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) activeNetworkNamespaceHandle.Close() return errors.New("Failed to get " + KubeTunnelIf + " tunnel interface handle due to " + err.Error()) } - glog.V(2).Infof("Successfully created tunnel interface " + KubeTunnelIf + " in endpoint " + endpointIP + ".") + klog.V(2).Infof("Successfully created tunnel interface " + KubeTunnelIf + " in endpoint " + endpointIP + ".") } // bring the tunnel interface up @@ -1247,7 +1247,7 @@ func (ln *linuxNetworking) prepareEndpointForDsrWithCRI(runtimeEndpoint, contain return errors.New("Failed to set hostNetworkNamespace handle due to " + err.Error()) } activeNetworkNamespaceHandle, err = netns.Get() - glog.Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) + klog.Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) activeNetworkNamespaceHandle.Close() return errors.New("Failed to bring up ipip tunnel interface in endpoint namespace due to " + err.Error()) } @@ -1264,11 +1264,11 @@ func (ln *linuxNetworking) prepareEndpointForDsrWithCRI(runtimeEndpoint, contain return errors.New("Failed to get activeNetworkNamespace handle due to " + err.Error()) } - glog.V(2).Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) + klog.V(2).Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) activeNetworkNamespaceHandle.Close() return errors.New("Failed to assign vip " + vip + " to kube-tunnel-if interface ") } - glog.Infof("Successfully assinged VIP: " + vip + " in endpoint " + endpointIP + ".") + klog.Infof("Successfully assinged VIP: " + vip + " in endpoint " + endpointIP + ".") // disable rp_filter on all interface err = ioutil.WriteFile("/proc/sys/net/ipv4/conf/kube-tunnel-if/rp_filter", []byte(strconv.Itoa(0)), 0640) @@ -1282,7 +1282,7 @@ func (ln *linuxNetworking) prepareEndpointForDsrWithCRI(runtimeEndpoint, contain return errors.New("Failed to get activeNetworkNamespace handle due to " + err.Error()) } - glog.Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) + klog.Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) activeNetworkNamespaceHandle.Close() return errors.New("Failed to disable rp_filter on kube-tunnel-if in the endpoint container") } @@ -1297,7 +1297,7 @@ func (ln *linuxNetworking) prepareEndpointForDsrWithCRI(runtimeEndpoint, contain if err != nil { return errors.New("Failed to get activeNetworkNamespace handle due to " + err.Error()) } - glog.Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) + klog.Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) activeNetworkNamespaceHandle.Close() return errors.New("Failed to disable rp_filter on eth0 in the endpoint container") } @@ -1312,12 +1312,12 @@ func (ln *linuxNetworking) prepareEndpointForDsrWithCRI(runtimeEndpoint, contain if err != nil { return errors.New("Failed to get activeNetworkNamespace handle due to " + err.Error()) } - glog.V(2).Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) + klog.V(2).Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) activeNetworkNamespaceHandle.Close() return errors.New("Failed to disable rp_filter on `all` in the endpoint container") } - glog.Infof("Successfully disabled rp_filter in endpoint " + endpointIP + ".") + klog.Infof("Successfully disabled rp_filter in endpoint " + endpointIP + ".") err = netns.Set(hostNetworkNamespaceHandle) if err != nil { @@ -1327,7 +1327,7 @@ func (ln *linuxNetworking) prepareEndpointForDsrWithCRI(runtimeEndpoint, contain if err != nil { return errors.New("Failed to get activeNetworkNamespace handle due to " + err.Error()) } - glog.Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) + klog.Infof("Current network namespace after revert namespace to host network namespace: " + activeNetworkNamespaceHandle.String()) activeNetworkNamespaceHandle.Close() return nil } @@ -1338,12 +1338,12 @@ func (nsc *NetworkServicesController) buildServicesInfo() serviceInfoMap { svc := obj.(*api.Service) if utils.ClusterIPIsNoneOrBlank(svc.Spec.ClusterIP) { - glog.V(2).Infof("Skipping service name:%s namespace:%s as there is no cluster IP", svc.Name, svc.Namespace) + klog.V(2).Infof("Skipping service name:%s namespace:%s as there is no cluster IP", svc.Name, svc.Namespace) continue } if svc.Spec.Type == "ExternalName" { - glog.V(2).Infof("Skipping service name:%s namespace:%s due to service Type=%s", svc.Name, svc.Namespace, svc.Spec.Type) + klog.V(2).Infof("Skipping service name:%s namespace:%s due to service Type=%s", svc.Name, svc.Namespace, svc.Spec.Type) continue } @@ -1491,7 +1491,7 @@ func (nsc *NetworkServicesController) ensureMasqueradeIptablesRule() error { return errors.New("Failed to delete iptables rule to masquerade all outbound IPVS traffic: " + err.Error() + ". Masquerade might still work...") } - glog.Infof("Deleted iptables rule to masquerade all outbound IVPS traffic.") + klog.Infof("Deleted iptables rule to masquerade all outbound IVPS traffic.") } } if len(nsc.podCidr) > 0 { @@ -1507,7 +1507,7 @@ func (nsc *NetworkServicesController) ensureMasqueradeIptablesRule() error { return errors.New("Failed to run iptables command" + err.Error()) } } - glog.V(2).Info("Successfully synced iptables masquerade rule") + klog.V(2).Info("Successfully synced iptables masquerade rule") return nil } @@ -1547,7 +1547,7 @@ func (nsc *NetworkServicesController) deleteBadMasqueradeIptablesRules() error { "Masquerade all might still work, or bugs may persist after upgrade...", err) } - glog.Infof("Deleted old/bad iptables rule to masquerade outbound traffic.") + klog.Infof("Deleted old/bad iptables rule to masquerade outbound traffic.") } } @@ -1585,7 +1585,7 @@ func (nsc *NetworkServicesController) syncHairpinIptablesRules() error { // Cleanup (if needed) and return if there's no hairpin-mode Services if len(rulesNeeded) == 0 { - glog.V(1).Info("No hairpin-mode enabled services found -- no hairpin rules created") + klog.V(1).Info("No hairpin-mode enabled services found -- no hairpin rules created") err := deleteHairpinIptablesRules() if err != nil { return errors.New("Error deleting hairpin rules: " + err.Error()) @@ -1665,16 +1665,16 @@ func (nsc *NetworkServicesController) syncHairpinIptablesRules() error { err = iptablesCmdHandler.Delete("nat", hairpinChain, args...) if err != nil { - glog.Errorf("Unable to delete hairpin rule \"%s\" from chain %s: %e", ruleFromNode, hairpinChain, err) + klog.Errorf("Unable to delete hairpin rule \"%s\" from chain %s: %e", ruleFromNode, hairpinChain, err) } else { - glog.V(1).Infof("Deleted invalid/outdated hairpin rule \"%s\" from chain %s", ruleFromNode, hairpinChain) + klog.V(1).Infof("Deleted invalid/outdated hairpin rule \"%s\" from chain %s", ruleFromNode, hairpinChain) } } else { // Ignore the chain creation rule if ruleFromNode == "-N "+hairpinChain { continue } - glog.V(1).Infof("Not removing invalid hairpin rule \"%s\" from chain %s", ruleFromNode, hairpinChain) + klog.V(1).Infof("Not removing invalid hairpin rule \"%s\" from chain %s", ruleFromNode, hairpinChain) } } } @@ -1738,9 +1738,9 @@ func deleteHairpinIptablesRules() error { if hasHairpinJumpRule { err = iptablesCmdHandler.Delete("nat", "POSTROUTING", jumpArgs...) if err != nil { - glog.Errorf("Unable to delete hairpin jump rule from chain \"POSTROUTING\": %e", err) + klog.Errorf("Unable to delete hairpin jump rule from chain \"POSTROUTING\": %e", err) } else { - glog.V(1).Info("Deleted hairpin jump rule from chain \"POSTROUTING\"") + klog.V(1).Info("Deleted hairpin jump rule from chain \"POSTROUTING\"") } } @@ -1773,7 +1773,7 @@ func deleteMasqueradeIptablesRule() error { if err != nil { return errors.New("Failed to run iptables command" + err.Error()) } - glog.V(2).Infof("Deleted iptables masquerade rule: %s", rule) + klog.V(2).Infof("Deleted iptables masquerade rule: %s", rule) break } } @@ -1892,7 +1892,7 @@ func (ln *linuxNetworking) ipvsAddService(svcs []*ipvs.Service, vip net.IP, prot if err != nil { return nil, err } - glog.V(2).Infof("Updated persistence/session-affinity for service: %s", ipvsServiceString(svc)) + klog.V(2).Infof("Updated persistence/session-affinity for service: %s", ipvsServiceString(svc)) } if changedIpvsSchedFlags(svc, flags) { @@ -1902,7 +1902,7 @@ func (ln *linuxNetworking) ipvsAddService(svcs []*ipvs.Service, vip net.IP, prot if err != nil { return nil, err } - glog.V(2).Infof("Updated scheduler flags for service: %s", ipvsServiceString(svc)) + klog.V(2).Infof("Updated scheduler flags for service: %s", ipvsServiceString(svc)) } if scheduler != svc.SchedName { @@ -1911,10 +1911,10 @@ func (ln *linuxNetworking) ipvsAddService(svcs []*ipvs.Service, vip net.IP, prot if err != nil { return nil, errors.New("Failed to update the scheduler for the service due to " + err.Error()) } - glog.V(2).Infof("Updated schedule for the service: %s", ipvsServiceString(svc)) + klog.V(2).Infof("Updated schedule for the service: %s", ipvsServiceString(svc)) } // TODO: Make this debug output when we get log levels - // glog.Fatal("ipvs service %s:%s:%s already exists so returning", vip.String(), + // klog.Fatal("ipvs service %s:%s:%s already exists so returning", vip.String(), // protocol, strconv.Itoa(int(port))) return svc, nil @@ -1936,7 +1936,7 @@ func (ln *linuxNetworking) ipvsAddService(svcs []*ipvs.Service, vip net.IP, prot if err != nil { return nil, err } - glog.V(1).Infof("Successfully added service: %s", ipvsServiceString(&svc)) + klog.V(1).Infof("Successfully added service: %s", ipvsServiceString(&svc)) return &svc, nil } @@ -1988,7 +1988,7 @@ func (ln *linuxNetworking) ipvsAddFWMarkService(vip net.IP, protocol, port uint1 if err != nil { return nil, err } - glog.V(2).Infof("Updated persistence/session-affinity for service: %s", ipvsServiceString(svc)) + klog.V(2).Infof("Updated persistence/session-affinity for service: %s", ipvsServiceString(svc)) } if changedIpvsSchedFlags(svc, flags) { @@ -1998,7 +1998,7 @@ func (ln *linuxNetworking) ipvsAddFWMarkService(vip net.IP, protocol, port uint1 if err != nil { return nil, err } - glog.V(2).Infof("Updated scheduler flags for service: %s", ipvsServiceString(svc)) + klog.V(2).Infof("Updated scheduler flags for service: %s", ipvsServiceString(svc)) } if scheduler != svc.SchedName { @@ -2007,10 +2007,10 @@ func (ln *linuxNetworking) ipvsAddFWMarkService(vip net.IP, protocol, port uint1 if err != nil { return nil, errors.New("Failed to update the scheduler for the service due to " + err.Error()) } - glog.V(2).Infof("Updated schedule for the service: %s", ipvsServiceString(svc)) + klog.V(2).Infof("Updated schedule for the service: %s", ipvsServiceString(svc)) } // TODO: Make this debug output when we get log levels - // glog.Fatal("ipvs service %s:%s:%s already exists so returning", vip.String(), + // klog.Fatal("ipvs service %s:%s:%s already exists so returning", vip.String(), // protocol, strconv.Itoa(int(port))) return svc, nil @@ -2032,14 +2032,14 @@ func (ln *linuxNetworking) ipvsAddFWMarkService(vip net.IP, protocol, port uint1 if err != nil { return nil, err } - glog.Infof("Successfully added service: %s", ipvsServiceString(&svc)) + klog.Infof("Successfully added service: %s", ipvsServiceString(&svc)) return &svc, nil } func (ln *linuxNetworking) ipvsAddServer(service *ipvs.Service, dest *ipvs.Destination) error { err := ln.ipvsNewDestination(service, dest) if err == nil { - glog.V(2).Infof("Successfully added destination %s to the service %s", + klog.V(2).Infof("Successfully added destination %s to the service %s", ipvsDestinationString(dest), ipvsServiceString(service)) return nil } @@ -2051,7 +2051,7 @@ func (ln *linuxNetworking) ipvsAddServer(service *ipvs.Service, dest *ipvs.Desti ipvsDestinationString(dest), ipvsServiceString(service), err.Error()) } // TODO: Make this debug output when we get log levels - // glog.Infof("ipvs destination %s already exists in the ipvs service %s so not adding destination", + // klog.Infof("ipvs destination %s already exists in the ipvs service %s so not adding destination", // ipvsDestinationString(dest), ipvsServiceString(service)) } else { return fmt.Errorf("Failed to add ipvs destination %s to the ipvs service %s due to : %s", @@ -2192,7 +2192,7 @@ func (ln *linuxNetworking) setupRoutesForExternalIPForDSR(serviceInfoMap service if !(strings.Contains(string(out), externalIPRouteTableName) || strings.Contains(string(out), externalIPRouteTableID)) { err = exec.Command("ip", "rule", "add", "prio", "32765", "from", "all", "lookup", externalIPRouteTableID).Run() if err != nil { - glog.Infof("Failed to add policy rule `ip rule add prio 32765 from all lookup external_ip` due to " + err.Error()) + klog.Infof("Failed to add policy rule `ip rule add prio 32765 from all lookup external_ip` due to " + err.Error()) return errors.New("Failed to add policy rule `ip rule add prio 32765 from all lookup external_ip` due to " + err.Error()) } } @@ -2204,7 +2204,7 @@ func (ln *linuxNetworking) setupRoutesForExternalIPForDSR(serviceInfoMap service for _, externalIP := range svc.externalIPs { // Verify the DSR annotation exists if !svc.directServerReturn { - glog.V(1).Infof("Skipping service %s/%s as it does not have DSR annotation\n", svc.namespace, svc.name) + klog.V(1).Infof("Skipping service %s/%s as it does not have DSR annotation\n", svc.namespace, svc.name) continue } @@ -2213,7 +2213,7 @@ func (ln *linuxNetworking) setupRoutesForExternalIPForDSR(serviceInfoMap service if !strings.Contains(outStr, externalIP) { if err = exec.Command("ip", "route", "add", externalIP, "dev", "kube-bridge", "table", externalIPRouteTableID).Run(); err != nil { - glog.Error("Failed to add route for " + externalIP + " in custom route table for external IP's due to: " + err.Error()) + klog.Error("Failed to add route for " + externalIP + " in custom route table for external IP's due to: " + err.Error()) continue } } @@ -2230,7 +2230,7 @@ func (ln *linuxNetworking) setupRoutesForExternalIPForDSR(serviceInfoMap service args := []string{"route", "del", "table", externalIPRouteTableID} args = append(args, route...) if err = exec.Command("ip", args...).Run(); err != nil { - glog.Errorf("Failed to del route for %v in custom route table for external IP's due to: %s", ip, err) + klog.Errorf("Failed to del route for %v in custom route table for external IP's due to: %s", ip, err) continue } } @@ -2293,7 +2293,7 @@ func (ln *linuxNetworking) getKubeDummyInterface() (netlink.Link, error) { var dummyVipInterface netlink.Link dummyVipInterface, err := netlink.LinkByName(KubeDummyIf) if err != nil && err.Error() == IfaceNotFound { - glog.V(1).Infof("Could not find dummy interface: " + KubeDummyIf + " to assign cluster ip's, creating one") + klog.V(1).Infof("Could not find dummy interface: " + KubeDummyIf + " to assign cluster ip's, creating one") err = netlink.LinkAdd(&netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: KubeDummyIf}}) if err != nil { return nil, errors.New("Failed to add dummy interface: " + err.Error()) @@ -2313,11 +2313,11 @@ func (ln *linuxNetworking) getKubeDummyInterface() (netlink.Link, error) { // Cleanup cleans all the configurations (IPVS, iptables, links) done func (nsc *NetworkServicesController) Cleanup() { // cleanup ipvs rules by flush - glog.Infof("Cleaning up IPVS configuration permanently") + klog.Infof("Cleaning up IPVS configuration permanently") handle, err := ipvs.New("") if err != nil { - glog.Errorf("Failed to cleanup ipvs rules: %s", err.Error()) + klog.Errorf("Failed to cleanup ipvs rules: %s", err.Error()) return } @@ -2326,14 +2326,14 @@ func (nsc *NetworkServicesController) Cleanup() { // cleanup iptables masquerade rule err = deleteMasqueradeIptablesRule() if err != nil { - glog.Errorf("Failed to cleanup iptablesmasquerade rule due to: %s", err.Error()) + klog.Errorf("Failed to cleanup iptablesmasquerade rule due to: %s", err.Error()) return } // cleanup iptables hairpin rules err = deleteHairpinIptablesRules() if err != nil { - glog.Errorf("Failed to cleanup iptables hairpin rules: %s", err.Error()) + klog.Errorf("Failed to cleanup iptables hairpin rules: %s", err.Error()) return } @@ -2343,16 +2343,16 @@ func (nsc *NetworkServicesController) Cleanup() { dummyVipInterface, err := netlink.LinkByName(KubeDummyIf) if err != nil { if err.Error() != IfaceNotFound { - glog.Infof("Dummy interface: " + KubeDummyIf + " does not exist") + klog.Infof("Dummy interface: " + KubeDummyIf + " does not exist") } } else { err = netlink.LinkDel(dummyVipInterface) if err != nil { - glog.Errorf("Could not delete dummy interface " + KubeDummyIf + " due to " + err.Error()) + klog.Errorf("Could not delete dummy interface " + KubeDummyIf + " due to " + err.Error()) return } } - glog.Infof("Successfully cleaned the ipvs configuration done by kube-router") + klog.Infof("Successfully cleaned the ipvs configuration done by kube-router") } func (nsc *NetworkServicesController) newEndpointsEventHandler() cache.ResourceEventHandler { @@ -2373,7 +2373,7 @@ func (nsc *NetworkServicesController) newEndpointsEventHandler() cache.ResourceE func (nsc *NetworkServicesController) handleEndpointsAdd(obj interface{}) { endpoints, ok := obj.(*api.Endpoints) if !ok { - glog.Errorf("unexpected object type: %v", obj) + klog.Errorf("unexpected object type: %v", obj) return } nsc.OnEndpointsUpdate(endpoints) @@ -2382,12 +2382,12 @@ func (nsc *NetworkServicesController) handleEndpointsAdd(obj interface{}) { func (nsc *NetworkServicesController) handleEndpointsUpdate(oldObj, newObj interface{}) { _, ok := oldObj.(*api.Endpoints) if !ok { - glog.Errorf("unexpected object type: %v", oldObj) + klog.Errorf("unexpected object type: %v", oldObj) return } newEndpoints, ok := newObj.(*api.Endpoints) if !ok { - glog.Errorf("unexpected object type: %v", newObj) + klog.Errorf("unexpected object type: %v", newObj) return } nsc.OnEndpointsUpdate(newEndpoints) @@ -2398,11 +2398,11 @@ func (nsc *NetworkServicesController) handleEndpointsDelete(obj interface{}) { if !ok { tombstone, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { - glog.Errorf("unexpected object type: %v", obj) + klog.Errorf("unexpected object type: %v", obj) return } if endpoints, ok = tombstone.Obj.(*api.Endpoints); !ok { - glog.Errorf("unexpected object type: %v", obj) + klog.Errorf("unexpected object type: %v", obj) return } } @@ -2426,7 +2426,7 @@ func (nsc *NetworkServicesController) newSvcEventHandler() cache.ResourceEventHa func (nsc *NetworkServicesController) handleServiceAdd(obj interface{}) { service, ok := obj.(*api.Service) if !ok { - glog.Errorf("unexpected object type: %v", obj) + klog.Errorf("unexpected object type: %v", obj) return } nsc.OnServiceUpdate(service) @@ -2435,12 +2435,12 @@ func (nsc *NetworkServicesController) handleServiceAdd(obj interface{}) { func (nsc *NetworkServicesController) handleServiceUpdate(oldObj, newObj interface{}) { _, ok := oldObj.(*api.Service) if !ok { - glog.Errorf("unexpected object type: %v", oldObj) + klog.Errorf("unexpected object type: %v", oldObj) return } newService, ok := newObj.(*api.Service) if !ok { - glog.Errorf("unexpected object type: %v", newObj) + klog.Errorf("unexpected object type: %v", newObj) return } nsc.OnServiceUpdate(newService) @@ -2451,11 +2451,11 @@ func (nsc *NetworkServicesController) handleServiceDelete(obj interface{}) { if !ok { tombstone, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { - glog.Errorf("unexpected object type: %v", obj) + klog.Errorf("unexpected object type: %v", obj) return } if service, ok = tombstone.Obj.(*api.Service); !ok { - glog.Errorf("unexpected object type: %v", obj) + klog.Errorf("unexpected object type: %v", obj) return } } diff --git a/pkg/controllers/proxy/service_endpoints_sync.go b/pkg/controllers/proxy/service_endpoints_sync.go index 193407b20f..92956e1feb 100644 --- a/pkg/controllers/proxy/service_endpoints_sync.go +++ b/pkg/controllers/proxy/service_endpoints_sync.go @@ -13,10 +13,10 @@ import ( "github.com/cloudnativelabs/kube-router/pkg/cri" "github.com/cloudnativelabs/kube-router/pkg/metrics" "github.com/cloudnativelabs/kube-router/pkg/utils" - "github.com/golang/glog" "github.com/moby/ipvs" "github.com/vishvananda/netlink" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" ) // sync the ipvs service and server details configured to reflect the desired state of Kubernetes services @@ -28,7 +28,7 @@ func (nsc *NetworkServicesController) syncIpvsServices(serviceInfoMap serviceInf if nsc.MetricsEnabled { metrics.ControllerIpvsServicesSyncTime.Observe(endTime.Seconds()) } - glog.V(1).Infof("sync ipvs services took %v", endTime) + klog.V(1).Infof("sync ipvs services took %v", endTime) }() var err error @@ -41,43 +41,43 @@ func (nsc *NetworkServicesController) syncIpvsServices(serviceInfoMap serviceInf err = nsc.setupClusterIPServices(serviceInfoMap, endpointsInfoMap, activeServiceEndpointMap) if err != nil { syncErrors = true - glog.Errorf("Error setting up IPVS services for service cluster IP's: %s", err.Error()) + klog.Errorf("Error setting up IPVS services for service cluster IP's: %s", err.Error()) } err = nsc.setupNodePortServices(serviceInfoMap, endpointsInfoMap, activeServiceEndpointMap) if err != nil { syncErrors = true - glog.Errorf("Error setting up IPVS services for service nodeport's: %s", err.Error()) + klog.Errorf("Error setting up IPVS services for service nodeport's: %s", err.Error()) } err = nsc.setupExternalIPServices(serviceInfoMap, endpointsInfoMap, activeServiceEndpointMap) if err != nil { syncErrors = true - glog.Errorf("Error setting up IPVS services for service external IP's and load balancer IP's: %s", err.Error()) + klog.Errorf("Error setting up IPVS services for service external IP's and load balancer IP's: %s", err.Error()) } err = nsc.cleanupStaleVIPs(activeServiceEndpointMap) if err != nil { syncErrors = true - glog.Errorf("Error cleaning up stale VIP's configured on the dummy interface: %s", err.Error()) + klog.Errorf("Error cleaning up stale VIP's configured on the dummy interface: %s", err.Error()) } err = nsc.cleanupStaleIPVSConfig(activeServiceEndpointMap) if err != nil { syncErrors = true - glog.Errorf("Error cleaning up stale IPVS services and servers: %s", err.Error()) + klog.Errorf("Error cleaning up stale IPVS services and servers: %s", err.Error()) } err = nsc.syncIpvsFirewall() if err != nil { syncErrors = true - glog.Errorf("Error syncing ipvs svc iptables rules to permit traffic to service VIP's: %s", err.Error()) + klog.Errorf("Error syncing ipvs svc iptables rules to permit traffic to service VIP's: %s", err.Error()) } err = nsc.setupForDSR(serviceInfoMap) if err != nil { syncErrors = true - glog.Errorf("Error setting up necessary policy based routing configuration needed for direct server return: %s", err.Error()) + klog.Errorf("Error setting up necessary policy based routing configuration needed for direct server return: %s", err.Error()) } if syncErrors { - glog.V(1).Info("One or more errors encountered during sync of IPVS services and servers to desired state") + klog.V(1).Info("One or more errors encountered during sync of IPVS services and servers to desired state") } else { - glog.V(1).Info("IPVS servers and services are synced to desired state") + klog.V(1).Info("IPVS servers and services are synced to desired state") } return nil @@ -114,7 +114,7 @@ func (nsc *NetworkServicesController) setupClusterIPServices(serviceInfoMap serv // create IPVS service for the service to be exposed through the cluster ip ipvsClusterVipSvc, err := nsc.ln.ipvsAddService(ipvsSvcs, svc.clusterIP, protocol, uint16(svc.port), svc.sessionAffinity, svc.sessionAffinityTimeoutSeconds, svc.scheduler, svc.flags) if err != nil { - glog.Errorf("Failed to create ipvs service for cluster ip: %s", err.Error()) + klog.Errorf("Failed to create ipvs service for cluster ip: %s", err.Error()) continue } var clusterServiceID = generateIPPortID(svc.clusterIP.String(), svc.protocol, strconv.Itoa(svc.port)) @@ -140,7 +140,7 @@ func (nsc *NetworkServicesController) setupClusterIPServices(serviceInfoMap serv err := nsc.ln.ipvsAddServer(ipvsClusterVipSvc, &dst) if err != nil { - glog.Errorf(err.Error()) + klog.Errorf(err.Error()) } else { activeServiceEndpointMap[clusterServiceID] = append(activeServiceEndpointMap[clusterServiceID], generateEndpointID(endpoint.ip, strconv.Itoa(endpoint.port))) } @@ -172,7 +172,7 @@ func (nsc *NetworkServicesController) setupNodePortServices(serviceInfoMap servi } endpoints := endpointsInfoMap[k] if svc.local && !hasActiveEndpoints(svc, endpoints) { - glog.V(1).Infof("Skipping setting up NodePort service %s/%s as it does not have active endpoints\n", svc.namespace, svc.name) + klog.V(1).Infof("Skipping setting up NodePort service %s/%s as it does not have active endpoints\n", svc.namespace, svc.name) continue } @@ -186,12 +186,12 @@ func (nsc *NetworkServicesController) setupNodePortServices(serviceInfoMap servi addrs, err := getAllLocalIPs() if err != nil { - glog.Errorf("Could not get list of system addresses for ipvs services: %s", err.Error()) + klog.Errorf("Could not get list of system addresses for ipvs services: %s", err.Error()) continue } if len(addrs) == 0 { - glog.Errorf("No IP addresses returned for nodeport service creation!") + klog.Errorf("No IP addresses returned for nodeport service creation!") continue } @@ -201,7 +201,7 @@ func (nsc *NetworkServicesController) setupNodePortServices(serviceInfoMap servi for i, addr := range addrs { ipvsNodeportSvcs[i], err = nsc.ln.ipvsAddService(ipvsSvcs, addr.IP, protocol, uint16(svc.nodePort), svc.sessionAffinity, svc.sessionAffinityTimeoutSeconds, svc.scheduler, svc.flags) if err != nil { - glog.Errorf("Failed to create ipvs service for node port due to: %s", err.Error()) + klog.Errorf("Failed to create ipvs service for node port due to: %s", err.Error()) continue } @@ -212,7 +212,7 @@ func (nsc *NetworkServicesController) setupNodePortServices(serviceInfoMap servi ipvsNodeportSvcs = make([]*ipvs.Service, 1) ipvsNodeportSvcs[0], err = nsc.ln.ipvsAddService(ipvsSvcs, nsc.nodeIP, protocol, uint16(svc.nodePort), svc.sessionAffinity, svc.sessionAffinityTimeoutSeconds, svc.scheduler, svc.flags) if err != nil { - glog.Errorf("Failed to create ipvs service for node port due to: %s", err.Error()) + klog.Errorf("Failed to create ipvs service for node port due to: %s", err.Error()) continue } @@ -232,7 +232,7 @@ func (nsc *NetworkServicesController) setupNodePortServices(serviceInfoMap servi if !svc.local || (svc.local && endpoint.isLocal) { err := nsc.ln.ipvsAddServer(ipvsNodeportSvcs[i], &dst) if err != nil { - glog.Errorf(err.Error()) + klog.Errorf(err.Error()) } else { activeServiceEndpointMap[nodeServiceIds[i]] = append(activeServiceEndpointMap[nodeServiceIds[i]], generateEndpointID(endpoint.ip, strconv.Itoa(endpoint.port))) } @@ -284,13 +284,13 @@ func (nsc *NetworkServicesController) setupExternalIPServices(serviceInfoMap ser } if svc.local && !hasActiveEndpoints(svc, endpoints) { - glog.V(1).Infof("Skipping setting up IPVS service for external IP and LoadBalancer IP for the service %s/%s as it does not have active endpoints\n", svc.namespace, svc.name) + klog.V(1).Infof("Skipping setting up IPVS service for external IP and LoadBalancer IP for the service %s/%s as it does not have active endpoints\n", svc.namespace, svc.name) continue } mangleTableRulesDump := bytes.Buffer{} mangleTableRules := []string{} if err := utils.SaveInto("mangle", &mangleTableRulesDump); err != nil { - glog.Errorf("Failed to run iptables-save: %s" + err.Error()) + klog.Errorf("Failed to run iptables-save: %s" + err.Error()) } else { mangleTableRules = strings.Split(mangleTableRulesDump.String(), "\n") } @@ -299,13 +299,13 @@ func (nsc *NetworkServicesController) setupExternalIPServices(serviceInfoMap ser if svc.directServerReturn && svc.directServerReturnMethod == "tunnel" { ipvsExternalIPSvc, err := nsc.ln.ipvsAddFWMarkService(net.ParseIP(externalIP), protocol, uint16(svc.port), svc.sessionAffinity, svc.sessionAffinityTimeoutSeconds, svc.scheduler, svc.flags) if err != nil { - glog.Errorf("Failed to create ipvs service for External IP: %s due to: %s", externalIP, err.Error()) + klog.Errorf("Failed to create ipvs service for External IP: %s due to: %s", externalIP, err.Error()) continue } externalIPServices = append(externalIPServices, externalIPService{ipvsSvc: ipvsExternalIPSvc, externalIP: externalIP}) fwMark, err := generateFwmark(externalIP, svc.protocol, strconv.Itoa(svc.port)) if err != nil { - glog.Errorf("Failed to generate Fwmark") + klog.Errorf("Failed to generate Fwmark") continue } externalIPServiceID = fmt.Sprint(fwMark) @@ -313,20 +313,20 @@ func (nsc *NetworkServicesController) setupExternalIPServices(serviceInfoMap ser // ensure there is iptables mangle table rule to FWMARK the packet err = setupMangleTableRule(externalIP, svc.protocol, strconv.Itoa(svc.port), externalIPServiceID) if err != nil { - glog.Errorf("Failed to setup mangle table rule to FMWARD the traffic to external IP") + klog.Errorf("Failed to setup mangle table rule to FMWARD the traffic to external IP") continue } // ensure VIP less director. we dont assign VIP to any interface err = nsc.ln.ipAddrDel(dummyVipInterface, externalIP) if err != nil && err.Error() != IfaceHasNoAddr { - glog.Errorf("Failed to delete external ip address from dummyVipInterface due to %s", err) + klog.Errorf("Failed to delete external ip address from dummyVipInterface due to %s", err) continue } // do policy routing to deliver the packet locally so that IPVS can pick the packet err = routeVIPTrafficToDirector("0x" + fmt.Sprintf("%x", fwMark)) if err != nil { - glog.Errorf("Failed to setup ip rule to lookup traffic to external IP: %s through custom "+ + klog.Errorf("Failed to setup ip rule to lookup traffic to external IP: %s through custom "+ "route table due to %s", externalIP, err.Error()) continue } @@ -334,13 +334,13 @@ func (nsc *NetworkServicesController) setupExternalIPServices(serviceInfoMap ser // ensure director with vip assigned err := nsc.ln.ipAddrAdd(dummyVipInterface, externalIP, true) if err != nil && err.Error() != IfaceHasAddr { - glog.Errorf("Failed to assign external ip %s to dummy interface %s due to %s", externalIP, KubeDummyIf, err.Error()) + klog.Errorf("Failed to assign external ip %s to dummy interface %s due to %s", externalIP, KubeDummyIf, err.Error()) } // create IPVS service for the service to be exposed through the external ip ipvsExternalIPSvc, err := nsc.ln.ipvsAddService(ipvsSvcs, net.ParseIP(externalIP), protocol, uint16(svc.port), svc.sessionAffinity, svc.sessionAffinityTimeoutSeconds, svc.scheduler, svc.flags) if err != nil { - glog.Errorf("Failed to create ipvs service for external ip: %s due to %s", externalIP, err.Error()) + klog.Errorf("Failed to create ipvs service for external ip: %s due to %s", externalIP, err.Error()) continue } externalIPServices = append(externalIPServices, externalIPService{ipvsSvc: ipvsExternalIPSvc, externalIP: externalIP}) @@ -349,7 +349,7 @@ func (nsc *NetworkServicesController) setupExternalIPServices(serviceInfoMap ser // ensure there is NO iptables mangle table rule to FWMARK the packet fwmark, err := generateFwmark(externalIP, svc.protocol, strconv.Itoa(svc.port)) if err != nil { - glog.Errorf("Failed to generate a fwmark due to " + err.Error()) + klog.Errorf("Failed to generate a fwmark due to " + err.Error()) continue } fwMark := fmt.Sprint(fwmark) @@ -357,7 +357,7 @@ func (nsc *NetworkServicesController) setupExternalIPServices(serviceInfoMap ser if strings.Contains(mangleTableRule, externalIP) && strings.Contains(mangleTableRule, fwMark) { err = nsc.ln.cleanupMangleTableRule(externalIP, svc.protocol, strconv.Itoa(svc.port), fwMark) if err != nil { - glog.Errorf("Failed to verify and cleanup any mangle table rule to FMWARD the traffic to external IP due to " + err.Error()) + klog.Errorf("Failed to verify and cleanup any mangle table rule to FMWARD the traffic to external IP due to " + err.Error()) continue } } @@ -393,7 +393,7 @@ func (nsc *NetworkServicesController) setupExternalIPServices(serviceInfoMap ser // add server to IPVS service err := nsc.ln.ipvsAddServer(externalIPService.ipvsSvc, &dst) if err != nil { - glog.Errorf(err.Error()) + klog.Errorf(err.Error()) } // For now just support IPVS tunnel mode, we can add other ways of DSR in future @@ -401,7 +401,7 @@ func (nsc *NetworkServicesController) setupExternalIPServices(serviceInfoMap ser podObj, err := nsc.getPodObjectForEndpoint(endpoint.ip) if err != nil { - glog.Errorf("Failed to find endpoint with ip: " + endpoint.ip + ". so skipping peparing endpoint for DSR") + klog.Errorf("Failed to find endpoint with ip: " + endpoint.ip + ". so skipping peparing endpoint for DSR") continue } @@ -413,12 +413,12 @@ func (nsc *NetworkServicesController) setupExternalIPServices(serviceInfoMap ser containerURL := podObj.Status.ContainerStatuses[0].ContainerID runtime, containerID, err := cri.EndpointParser(containerURL) if err != nil { - glog.Errorf("couldn't get containerID (container=%s, pod=%s). Skipping DSR endpoint set up", podObj.Spec.Containers[0].Name, podObj.Name) + klog.Errorf("couldn't get containerID (container=%s, pod=%s). Skipping DSR endpoint set up", podObj.Spec.Containers[0].Name, podObj.Name) continue } if containerID == "" { - glog.Errorf("Failed to find container id for the endpoint with ip: " + endpoint.ip + " so skipping peparing endpoint for DSR") + klog.Errorf("Failed to find container id for the endpoint with ip: " + endpoint.ip + " so skipping peparing endpoint for DSR") continue } @@ -426,14 +426,14 @@ func (nsc *NetworkServicesController) setupExternalIPServices(serviceInfoMap ser // WARN: This method is deprecated and will be removed once docker-shim is removed from kubelet. err = nsc.ln.prepareEndpointForDsr(containerID, endpoint.ip, externalIPService.externalIP) if err != nil { - glog.Errorf("Failed to prepare endpoint %s to do direct server return due to %s", endpoint.ip, err.Error()) + klog.Errorf("Failed to prepare endpoint %s to do direct server return due to %s", endpoint.ip, err.Error()) } } else { // We expect CRI compliant runtimes here // ugly workaround, refactoring of pkg/Proxy is required err = nsc.ln.(*linuxNetworking).prepareEndpointForDsrWithCRI(nsc.dsr.runtimeEndpoint, containerID, endpoint.ip, externalIPService.externalIP) if err != nil { - glog.Errorf("Failed to prepare endpoint %s to do DSR due to: %s", endpoint.ip, err.Error()) + klog.Errorf("Failed to prepare endpoint %s to do DSR due to: %s", endpoint.ip, err.Error()) } } } @@ -444,26 +444,26 @@ func (nsc *NetworkServicesController) setupExternalIPServices(serviceInfoMap ser } func (nsc *NetworkServicesController) setupForDSR(serviceInfoMap serviceInfoMap) error { - glog.V(1).Infof("Setting up policy routing required for Direct Server Return functionality.") + klog.V(1).Infof("Setting up policy routing required for Direct Server Return functionality.") err := nsc.ln.setupPolicyRoutingForDSR() if err != nil { return errors.New("Failed setup PBR for DSR due to: " + err.Error()) } - glog.V(1).Infof("Custom routing table " + customDSRRouteTableName + " required for Direct Server Return is setup as expected.") + klog.V(1).Infof("Custom routing table " + customDSRRouteTableName + " required for Direct Server Return is setup as expected.") - glog.V(1).Infof("Setting up custom route table required to add routes for external IP's.") + klog.V(1).Infof("Setting up custom route table required to add routes for external IP's.") err = nsc.ln.setupRoutesForExternalIPForDSR(serviceInfoMap) if err != nil { - glog.Errorf("Failed setup custom routing table required to add routes for external IP's due to: " + err.Error()) + klog.Errorf("Failed setup custom routing table required to add routes for external IP's due to: " + err.Error()) return errors.New("Failed setup custom routing table required to add routes for external IP's due to: " + err.Error()) } - glog.V(1).Infof("Custom routing table " + externalIPRouteTableName + " required for Direct Server Return is setup as expected.") + klog.V(1).Infof("Custom routing table " + externalIPRouteTableName + " required for Direct Server Return is setup as expected.") return nil } func (nsc *NetworkServicesController) cleanupStaleVIPs(activeServiceEndpointMap map[string][]string) error { // cleanup stale IPs on dummy interface - glog.V(1).Info("Cleaning up if any, old service IPs on dummy interface") + klog.V(1).Info("Cleaning up if any, old service IPs on dummy interface") addrActive := make(map[string]bool) for k := range activeServiceEndpointMap { // verify active and its a generateIPPortID() type service @@ -485,10 +485,10 @@ func (nsc *NetworkServicesController) cleanupStaleVIPs(activeServiceEndpointMap for _, addr := range addrs { isActive := addrActive[addr.IP.String()] if !isActive { - glog.V(1).Infof("Found an IP %s which is no longer needed so cleaning up", addr.IP.String()) + klog.V(1).Infof("Found an IP %s which is no longer needed so cleaning up", addr.IP.String()) err := nsc.ln.ipAddrDel(dummyVipInterface, addr.IP.String()) if err != nil { - glog.Errorf("Failed to delete stale IP %s due to: %s", + klog.Errorf("Failed to delete stale IP %s due to: %s", addr.IP.String(), err.Error()) continue } @@ -504,7 +504,7 @@ func (nsc *NetworkServicesController) cleanupStaleIPVSConfig(activeServiceEndpoi } // cleanup stale ipvs service and servers - glog.V(1).Info("Cleaning up if any, old ipvs service and servers which are no longer needed") + klog.V(1).Info("Cleaning up if any, old ipvs service and servers which are no longer needed") if err != nil { return errors.New("Failed to list IPVS services: " + err.Error()) @@ -538,23 +538,23 @@ func (nsc *NetworkServicesController) cleanupStaleIPVSConfig(activeServiceEndpoi } if excluded { - glog.V(1).Infof("Ignoring deletion of an IPVS service %s in an excluded cidr", + klog.V(1).Infof("Ignoring deletion of an IPVS service %s in an excluded cidr", ipvsServiceString(ipvsSvc)) continue } - glog.V(1).Infof("Found a IPVS service %s which is no longer needed so cleaning up", + klog.V(1).Infof("Found a IPVS service %s which is no longer needed so cleaning up", ipvsServiceString(ipvsSvc)) err := nsc.ln.ipvsDelService(ipvsSvc) if err != nil { - glog.Errorf("Failed to delete stale IPVS service %s due to: %s", + klog.Errorf("Failed to delete stale IPVS service %s due to: %s", ipvsServiceString(ipvsSvc), err.Error()) continue } } else { dsts, err := nsc.ln.ipvsGetDestinations(ipvsSvc) if err != nil { - glog.Errorf("Failed to get list of servers from ipvs service") + klog.Errorf("Failed to get list of servers from ipvs service") } for _, dst := range dsts { validEp := false @@ -565,11 +565,11 @@ func (nsc *NetworkServicesController) cleanupStaleIPVSConfig(activeServiceEndpoi } } if !validEp { - glog.V(1).Infof("Found a destination %s in service %s which is no longer needed so cleaning up", + klog.V(1).Infof("Found a destination %s in service %s which is no longer needed so cleaning up", ipvsDestinationString(dst), ipvsServiceString(ipvsSvc)) err = nsc.ipvsDeleteDestination(ipvsSvc, dst) if err != nil { - glog.Errorf("Failed to delete destination %s from ipvs service %s", + klog.Errorf("Failed to delete destination %s from ipvs service %s", ipvsDestinationString(dst), ipvsServiceString(ipvsSvc)) } } diff --git a/pkg/controllers/routing/aws.go b/pkg/controllers/routing/aws.go index 1b8810540f..446f986342 100644 --- a/pkg/controllers/routing/aws.go +++ b/pkg/controllers/routing/aws.go @@ -10,7 +10,7 @@ import ( "github.com/aws/aws-sdk-go/aws/ec2metadata" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" - "github.com/golang/glog" + "k8s.io/klog/v2" v1core "k8s.io/api/core/v1" ) @@ -30,7 +30,7 @@ func (nrc *NetworkRoutingController) disableSourceDestinationCheck() { providerID := strings.Replace(node.Spec.ProviderID, "///", "//", 1) URL, err := url.Parse(providerID) if err != nil { - glog.Errorf("Failed to parse URL for providerID " + providerID + " : " + err.Error()) + klog.Errorf("Failed to parse URL for providerID " + providerID + " : " + err.Error()) return } instanceID := URL.Path @@ -40,7 +40,7 @@ func (nrc *NetworkRoutingController) disableSourceDestinationCheck() { metadataClient := ec2metadata.New(sess) region, err := metadataClient.Region() if err != nil { - glog.Errorf("Failed to disable source destination check due to: " + err.Error()) + klog.Errorf("Failed to disable source destination check due to: " + err.Error()) return } sess.Config.Region = aws.String(region) @@ -57,12 +57,12 @@ func (nrc *NetworkRoutingController) disableSourceDestinationCheck() { awserr := err.(awserr.Error) if awserr.Code() == "UnauthorizedOperation" { nrc.ec2IamAuthorized = false - glog.Errorf("Node does not have necessary IAM creds to modify instance attribute. So skipping disabling src-dst check.") + klog.Errorf("Node does not have necessary IAM creds to modify instance attribute. So skipping disabling src-dst check.") return } - glog.Errorf("Failed to disable source destination check due to: %v", err.Error()) + klog.Errorf("Failed to disable source destination check due to: %v", err.Error()) } else { - glog.Infof("Disabled source destination check for the instance: " + instanceID) + klog.Infof("Disabled source destination check for the instance: " + instanceID) } // to prevent EC2 rejecting API call due to API throttling give a delay between the calls diff --git a/pkg/controllers/routing/bgp_peers.go b/pkg/controllers/routing/bgp_peers.go index 7af849582e..e96d536ea1 100644 --- a/pkg/controllers/routing/bgp_peers.go +++ b/pkg/controllers/routing/bgp_peers.go @@ -12,11 +12,11 @@ import ( "github.com/cloudnativelabs/kube-router/pkg/metrics" "github.com/cloudnativelabs/kube-router/pkg/options" "github.com/cloudnativelabs/kube-router/pkg/utils" - "github.com/golang/glog" gobgpapi "github.com/osrg/gobgp/api" gobgp "github.com/osrg/gobgp/pkg/server" v1core "k8s.io/api/core/v1" "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" ) // Refresh the peer relationship with rest of the nodes in the cluster (iBGP peers). Node add/remove @@ -33,7 +33,7 @@ func (nrc *NetworkRoutingController) syncInternalPeers() { if nrc.MetricsEnabled { metrics.ControllerBGPInternalPeersSyncTime.Observe(endTime.Seconds()) } - glog.V(2).Infof("Syncing BGP peers for the node took %v", endTime) + klog.V(2).Infof("Syncing BGP peers for the node took %v", endTime) }() // get the current list of the nodes from API server @@ -48,7 +48,7 @@ func (nrc *NetworkRoutingController) syncInternalPeers() { node := obj.(*v1core.Node) nodeIP, err := utils.GetNodeIP(node) if err != nil { - glog.Errorf("Failed to find a node IP and therefore cannot sync internal BGP Peer: %v", err) + klog.Errorf("Failed to find a node IP and therefore cannot sync internal BGP Peer: %v", err) continue } @@ -69,21 +69,21 @@ func (nrc *NetworkRoutingController) syncInternalPeers() { if !nrc.bgpFullMeshMode { nodeasn, ok := node.ObjectMeta.Annotations[nodeASNAnnotation] if !ok { - glog.Infof("Not peering with the Node %s as ASN number of the node is unknown.", + klog.Infof("Not peering with the Node %s as ASN number of the node is unknown.", nodeIP.String()) continue } asnNo, err := strconv.ParseUint(nodeasn, 0, 32) if err != nil { - glog.Infof("Not peering with the Node %s as ASN number of the node is invalid.", + klog.Infof("Not peering with the Node %s as ASN number of the node is invalid.", nodeIP.String()) continue } // if the nodes ASN number is different from ASN number of current node skip peering if nrc.nodeAsnNumber != uint32(asnNo) { - glog.Infof("Not peering with the Node %s as ASN number of the node is different.", + klog.Infof("Not peering with the Node %s as ASN number of the node is different.", nodeIP.String()) continue } @@ -153,7 +153,7 @@ func (nrc *NetworkRoutingController) syncInternalPeers() { Peer: n, }); err != nil { if !strings.Contains(err.Error(), "can't overwrite the existing peer") { - glog.Errorf("Failed to add node %s as peer due to %s", nodeIP.String(), err) + klog.Errorf("Failed to add node %s as peer due to %s", nodeIP.String(), err) } } } @@ -176,7 +176,7 @@ func (nrc *NetworkRoutingController) syncInternalPeers() { // delete the neighbor for the nodes that are removed for _, ip := range removedNodes { if err := nrc.bgpServer.DeletePeer(context.Background(), &gobgpapi.DeletePeerRequest{Address: ip}); err != nil { - glog.Errorf("Failed to remove node %s as peer due to %s", ip, err) + klog.Errorf("Failed to remove node %s as peer due to %s", ip, err) } delete(nrc.activeNodes, ip) } @@ -231,7 +231,7 @@ func connectToExternalBGPPeers(server *gobgp.BgpServer, peerNeighbors []*gobgpap return fmt.Errorf("Error peering with peer router "+ "%q due to: %s", n.Conf.NeighborAddress, err) } - glog.V(2).Infof("Successfully configured %s in ASN %v as BGP peer to the node", + klog.V(2).Infof("Successfully configured %s in ASN %v as BGP peer to the node", n.Conf.NeighborAddress, n.Conf.PeerAs) } return nil @@ -304,11 +304,11 @@ func (nrc *NetworkRoutingController) newNodeEventHandler() cache.ResourceEventHa node := obj.(*v1core.Node) nodeIP, err := utils.GetNodeIP(node) if err != nil { - glog.Errorf("New node received, but we were unable to add it as we were couldn't find it's node IP: %v", err) + klog.Errorf("New node received, but we were unable to add it as we were couldn't find it's node IP: %v", err) return } - glog.V(2).Infof("Received node %s added update from watch API so peer with new node", nodeIP) + klog.V(2).Infof("Received node %s added update from watch API so peer with new node", nodeIP) nrc.OnNodeUpdate(obj) }, UpdateFunc: func(oldObj, newObj interface{}) { @@ -319,11 +319,11 @@ func (nrc *NetworkRoutingController) newNodeEventHandler() cache.ResourceEventHa if !ok { tombstone, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { - glog.Errorf("unexpected object type: %v", obj) + klog.Errorf("unexpected object type: %v", obj) return } if node, ok = tombstone.Obj.(*v1core.Node); !ok { - glog.Errorf("unexpected object type: %v", obj) + klog.Errorf("unexpected object type: %v", obj) return } } @@ -331,9 +331,9 @@ func (nrc *NetworkRoutingController) newNodeEventHandler() cache.ResourceEventHa // In this case even if we can't get the NodeIP that's alright as the node is being removed anyway and // future node lister operations that happen in OnNodeUpdate won't be affected as the node won't be returned if err == nil { - glog.Infof("Received node %s removed update from watch API, so remove node from peer", nodeIP) + klog.Infof("Received node %s removed update from watch API, so remove node from peer", nodeIP) } else { - glog.Infof("Received node (IP unavailable) removed update from watch API, so remove node from peer") + klog.Infof("Received node (IP unavailable) removed update from watch API, so remove node from peer") } nrc.OnNodeUpdate(obj) @@ -352,7 +352,7 @@ func (nrc *NetworkRoutingController) OnNodeUpdate(obj interface{}) { // update export policies so that NeighborSet gets updated with new set of nodes err := nrc.AddPolicies() if err != nil { - glog.Errorf("Error adding BGP policies: %s", err.Error()) + klog.Errorf("Error adding BGP policies: %s", err.Error()) } if nrc.bgpEnableInternal { diff --git a/pkg/controllers/routing/bgp_policies.go b/pkg/controllers/routing/bgp_policies.go index 3c4ce9c4d9..e933187ca5 100644 --- a/pkg/controllers/routing/bgp_policies.go +++ b/pkg/controllers/routing/bgp_policies.go @@ -7,9 +7,9 @@ import ( "strconv" "strings" - "github.com/golang/glog" gobgpapi "github.com/osrg/gobgp/api" v1core "k8s.io/api/core/v1" + "k8s.io/klog/v2" "github.com/cloudnativelabs/kube-router/pkg/utils" ) @@ -23,27 +23,27 @@ func (nrc *NetworkRoutingController) AddPolicies() error { err := nrc.addPodCidrDefinedSet() if err != nil { - glog.Errorf("Failed to add `podcidrdefinedset` defined set: %s", err) + klog.Errorf("Failed to add `podcidrdefinedset` defined set: %s", err) } err = nrc.addServiceVIPsDefinedSet() if err != nil { - glog.Errorf("Failed to add `servicevipsdefinedset` defined set: %s", err) + klog.Errorf("Failed to add `servicevipsdefinedset` defined set: %s", err) } iBGPPeerCIDRs, err := nrc.addiBGPPeersDefinedSet() if err != nil { - glog.Errorf("Failed to add `iBGPpeerset` defined set: %s", err) + klog.Errorf("Failed to add `iBGPpeerset` defined set: %s", err) } externalBGPPeerCIDRs, err := nrc.addExternalBGPPeersDefinedSet() if err != nil { - glog.Errorf("Failed to add `externalpeerset` defined set: %s", err) + klog.Errorf("Failed to add `externalpeerset` defined set: %s", err) } err = nrc.addAllBGPPeersDefinedSet(iBGPPeerCIDRs, externalBGPPeerCIDRs) if err != nil { - glog.Errorf("Failed to add `allpeerset` defined set: %s", err) + klog.Errorf("Failed to add `allpeerset` defined set: %s", err) } err = nrc.addExportPolicies() @@ -174,7 +174,7 @@ func (nrc *NetworkRoutingController) addiBGPPeersDefinedSet() ([]string, error) nodeObj := node.(*v1core.Node) nodeIP, err := utils.GetNodeIP(nodeObj) if err != nil { - glog.Errorf("Failed to find a node IP and therefore cannot add internal BGP Peer: %v", err) + klog.Errorf("Failed to find a node IP and therefore cannot add internal BGP Peer: %v", err) continue } iBGPPeerCIDRs = append(iBGPPeerCIDRs, nodeIP.String()+"/32") diff --git a/pkg/controllers/routing/ecmp_vip.go b/pkg/controllers/routing/ecmp_vip.go index 04cf3dba23..5e35d45799 100644 --- a/pkg/controllers/routing/ecmp_vip.go +++ b/pkg/controllers/routing/ecmp_vip.go @@ -10,18 +10,18 @@ import ( "strings" - "github.com/golang/glog" "github.com/golang/protobuf/ptypes" "github.com/golang/protobuf/ptypes/any" gobgpapi "github.com/osrg/gobgp/api" v1core "k8s.io/api/core/v1" "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" ) // bgpAdvertiseVIP advertises the service vip (cluster ip or load balancer ip or external IP) the configured peers func (nrc *NetworkRoutingController) bgpAdvertiseVIP(vip string) error { - glog.V(2).Infof("Advertising route: '%s/%s via %s' to peers", vip, strconv.Itoa(32), nrc.nodeIP.String()) + klog.V(2).Infof("Advertising route: '%s/%s via %s' to peers", vip, strconv.Itoa(32), nrc.nodeIP.String()) a1, _ := ptypes.MarshalAny(&gobgpapi.OriginAttribute{ Origin: 0, @@ -47,7 +47,7 @@ func (nrc *NetworkRoutingController) bgpAdvertiseVIP(vip string) error { // bgpWithdrawVIP unadvertises the service vip func (nrc *NetworkRoutingController) bgpWithdrawVIP(vip string) error { - glog.V(2).Infof("Withdrawing route: '%s/%s via %s' to peers", vip, strconv.Itoa(32), nrc.nodeIP.String()) + klog.V(2).Infof("Withdrawing route: '%s/%s via %s' to peers", vip, strconv.Itoa(32), nrc.nodeIP.String()) a1, _ := ptypes.MarshalAny(&gobgpapi.OriginAttribute{ Origin: 0, @@ -77,7 +77,7 @@ func (nrc *NetworkRoutingController) advertiseVIPs(vips []string) { for _, vip := range vips { err := nrc.bgpAdvertiseVIP(vip) if err != nil { - glog.Errorf("error advertising IP: %q, error: %v", vip, err) + klog.Errorf("error advertising IP: %q, error: %v", vip, err) } } } @@ -86,7 +86,7 @@ func (nrc *NetworkRoutingController) withdrawVIPs(vips []string) { for _, vip := range vips { err := nrc.bgpWithdrawVIP(vip) if err != nil { - glog.Errorf("error withdrawing IP: %q, error: %v", vip, err) + klog.Errorf("error withdrawing IP: %q, error: %v", vip, err) } } } @@ -107,27 +107,27 @@ func (nrc *NetworkRoutingController) newServiceEventHandler() cache.ResourceEven func getServiceObject(obj interface{}) (svc *v1core.Service) { if svc, _ = obj.(*v1core.Service); svc == nil { - glog.Errorf("cache indexer returned obj that is not type *v1.Service") + klog.Errorf("cache indexer returned obj that is not type *v1.Service") } return } func (nrc *NetworkRoutingController) handleServiceUpdate(svc *v1core.Service) { if !nrc.bgpServerStarted { - glog.V(3).Infof("Skipping update to service: %s/%s, controller still performing bootup full-sync", svc.Namespace, svc.Name) + klog.V(3).Infof("Skipping update to service: %s/%s, controller still performing bootup full-sync", svc.Namespace, svc.Name) return } toAdvertise, toWithdraw, err := nrc.getActiveVIPs() if err != nil { - glog.Errorf("error getting routes for services: %s", err) + klog.Errorf("error getting routes for services: %s", err) return } // update export policies so that new VIP's gets added to clusteripprefixset and vip gets advertised to peers err = nrc.AddPolicies() if err != nil { - glog.Errorf("Error adding BGP policies: %s", err.Error()) + klog.Errorf("Error adding BGP policies: %s", err.Error()) } nrc.advertiseVIPs(toAdvertise) @@ -137,18 +137,18 @@ func (nrc *NetworkRoutingController) handleServiceUpdate(svc *v1core.Service) { func (nrc *NetworkRoutingController) handleServiceDelete(svc *v1core.Service) { if !nrc.bgpServerStarted { - glog.V(3).Infof("Skipping update to service: %s/%s, controller still performing bootup full-sync", svc.Namespace, svc.Name) + klog.V(3).Infof("Skipping update to service: %s/%s, controller still performing bootup full-sync", svc.Namespace, svc.Name) return } err := nrc.AddPolicies() if err != nil { - glog.Errorf("Error adding BGP policies: %s", err.Error()) + klog.Errorf("Error adding BGP policies: %s", err.Error()) } activeVIPs, _, err := nrc.getActiveVIPs() if err != nil { - glog.Errorf("Failed to get active VIP's on service delete event due to: %s", err.Error()) + klog.Errorf("Failed to get active VIP's on service delete event due to: %s", err.Error()) return } activeVIPsMap := make(map[string]bool) @@ -169,14 +169,14 @@ func (nrc *NetworkRoutingController) handleServiceDelete(svc *v1core.Service) { func (nrc *NetworkRoutingController) tryHandleServiceUpdate(obj interface{}, logMsgFormat string) { if svc := getServiceObject(obj); svc != nil { - glog.V(1).Infof(logMsgFormat, svc.Namespace, svc.Name) + klog.V(1).Infof(logMsgFormat, svc.Namespace, svc.Name) // If the service is headless and the previous version of the service is either non-existent or also headless, // skip processing as we only work with VIPs in the next section. Since the ClusterIP field is immutable we don't // need to consider previous versions of the service here as we are guaranteed if is a ClusterIP now, it was a // ClusterIP before. if utils.ServiceIsHeadless(obj) { - glog.V(1).Infof("%s/%s is headless, skipping...", svc.Namespace, svc.Name) + klog.V(1).Infof("%s/%s is headless, skipping...", svc.Namespace, svc.Name) return } @@ -189,19 +189,19 @@ func (nrc *NetworkRoutingController) tryHandleServiceDelete(obj interface{}, log if !ok { tombstone, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { - glog.Errorf("unexpected object type: %v", obj) + klog.Errorf("unexpected object type: %v", obj) return } if svc, ok = tombstone.Obj.(*v1core.Service); !ok { - glog.Errorf("unexpected object type: %v", obj) + klog.Errorf("unexpected object type: %v", obj) return } } - glog.V(1).Infof(logMsgFormat, svc.Namespace, svc.Name) + klog.V(1).Infof(logMsgFormat, svc.Namespace, svc.Name) // If the service is headless skip processing as we only work with VIPs in the next section. if utils.ServiceIsHeadless(obj) { - glog.V(1).Infof("%s/%s is headless, skipping...", svc.Namespace, svc.Name) + klog.V(1).Infof("%s/%s is headless, skipping...", svc.Namespace, svc.Name) return } @@ -274,7 +274,7 @@ func (nrc *NetworkRoutingController) newEndpointsEventHandler() cache.ResourceEv // Service was not created yet. func (nrc *NetworkRoutingController) OnEndpointsAdd(obj interface{}) { if !nrc.bgpServerStarted { - glog.V(3).Info("Skipping OnAdd event to endpoint, controller still performing bootup full-sync") + klog.V(3).Info("Skipping OnAdd event to endpoint, controller still performing bootup full-sync") return } @@ -285,7 +285,7 @@ func (nrc *NetworkRoutingController) OnEndpointsAdd(obj interface{}) { func (nrc *NetworkRoutingController) OnEndpointsUpdate(obj interface{}) { ep, ok := obj.(*v1core.Endpoints) if !ok { - glog.Errorf("cache indexer returned obj that is not type *v1.Endpoints") + klog.Errorf("cache indexer returned obj that is not type *v1.Endpoints") return } @@ -293,15 +293,15 @@ func (nrc *NetworkRoutingController) OnEndpointsUpdate(obj interface{}) { return } - glog.V(1).Infof("Received update to endpoint: %s/%s from watch API", ep.Namespace, ep.Name) + klog.V(1).Infof("Received update to endpoint: %s/%s from watch API", ep.Namespace, ep.Name) if !nrc.bgpServerStarted { - glog.V(3).Infof("Skipping update to endpoint: %s/%s, controller still performing bootup full-sync", ep.Namespace, ep.Name) + klog.V(3).Infof("Skipping update to endpoint: %s/%s, controller still performing bootup full-sync", ep.Namespace, ep.Name) return } svc, exists, err := utils.ServiceForEndpoints(&nrc.svcLister, ep) if err != nil { - glog.Errorf("failed to convert endpoints resource to service: %s", err) + klog.Errorf("failed to convert endpoints resource to service: %s", err) return } diff --git a/pkg/controllers/routing/network_routes_controller.go b/pkg/controllers/routing/network_routes_controller.go index 58461b633f..5d70fd0ca2 100644 --- a/pkg/controllers/routing/network_routes_controller.go +++ b/pkg/controllers/routing/network_routes_controller.go @@ -20,11 +20,11 @@ import ( "github.com/cloudnativelabs/kube-router/pkg/options" "github.com/cloudnativelabs/kube-router/pkg/utils" "github.com/coreos/go-iptables/iptables" - "github.com/golang/glog" "github.com/golang/protobuf/ptypes" "github.com/golang/protobuf/ptypes/any" gobgpapi "github.com/osrg/gobgp/api" gobgp "github.com/osrg/gobgp/pkg/server" + "k8s.io/klog/v2" "github.com/prometheus/client_golang/prometheus" "github.com/vishvananda/netlink" @@ -133,10 +133,10 @@ func (nrc *NetworkRoutingController) Run(healthChan chan<- *healthcheck.Controll nrc.updateCNIConfig() } - glog.V(1).Info("Populating ipsets.") + klog.V(1).Info("Populating ipsets.") err = nrc.syncNodeIPSets() if err != nil { - glog.Errorf("Failed initial ipset setup: %s", err) + klog.Errorf("Failed initial ipset setup: %s", err) } // In case of cluster provisioned on AWS disable source-destination check @@ -148,48 +148,48 @@ func (nrc *NetworkRoutingController) Run(healthChan chan<- *healthcheck.Controll // enable IP forwarding for the packets coming in/out from the pods err = nrc.enableForwarding() if err != nil { - glog.Errorf("Failed to enable IP forwarding of traffic from pods: %s", err.Error()) + klog.Errorf("Failed to enable IP forwarding of traffic from pods: %s", err.Error()) } nrc.CNIFirewallSetup.Broadcast() // Handle ipip tunnel overlay if nrc.enableOverlays { - glog.V(1).Info("IPIP Tunnel Overlay enabled in configuration.") - glog.V(1).Info("Setting up overlay networking.") + klog.V(1).Info("IPIP Tunnel Overlay enabled in configuration.") + klog.V(1).Info("Setting up overlay networking.") err = nrc.enablePolicyBasedRouting() if err != nil { - glog.Errorf("Failed to enable required policy based routing: %s", err.Error()) + klog.Errorf("Failed to enable required policy based routing: %s", err.Error()) } } else { - glog.V(1).Info("IPIP Tunnel Overlay disabled in configuration.") - glog.V(1).Info("Cleaning up old overlay networking if needed.") + klog.V(1).Info("IPIP Tunnel Overlay disabled in configuration.") + klog.V(1).Info("Cleaning up old overlay networking if needed.") err = nrc.disablePolicyBasedRouting() if err != nil { - glog.Errorf("Failed to disable policy based routing: %s", err.Error()) + klog.Errorf("Failed to disable policy based routing: %s", err.Error()) } } - glog.V(1).Info("Performing cleanup of depreciated rules/ipsets (if needed).") + klog.V(1).Info("Performing cleanup of depreciated rules/ipsets (if needed).") err = nrc.deleteBadPodEgressRules() if err != nil { - glog.Errorf("Error cleaning up old/bad Pod egress rules: %s", err.Error()) + klog.Errorf("Error cleaning up old/bad Pod egress rules: %s", err.Error()) } // Handle Pod egress masquerading configuration if nrc.enablePodEgress { - glog.V(1).Infoln("Enabling Pod egress.") + klog.V(1).Infoln("Enabling Pod egress.") err = nrc.createPodEgressRule() if err != nil { - glog.Errorf("Error enabling Pod egress: %s", err.Error()) + klog.Errorf("Error enabling Pod egress: %s", err.Error()) } } else { - glog.V(1).Infoln("Disabling Pod egress.") + klog.V(1).Infoln("Disabling Pod egress.") err = nrc.deletePodEgressRule() if err != nil { - glog.Warningf("Error cleaning up Pod Egress related networking: %s", err) + klog.Warningf("Error cleaning up Pod Egress related networking: %s", err) } } @@ -200,43 +200,43 @@ func (nrc *NetworkRoutingController) Run(healthChan chan<- *healthcheck.Controll linkAttrs.Name = "kube-bridge" bridge := &netlink.Bridge{LinkAttrs: linkAttrs} if err = netlink.LinkAdd(bridge); err != nil { - glog.Errorf("Failed to create `kube-router` bridge due to %s. Will be created by CNI bridge plugin when pod is launched.", err.Error()) + klog.Errorf("Failed to create `kube-router` bridge due to %s. Will be created by CNI bridge plugin when pod is launched.", err.Error()) } kubeBridgeIf, err = netlink.LinkByName("kube-bridge") if err != nil { - glog.Errorf("Failed to find created `kube-router` bridge due to %s. Will be created by CNI bridge plugin when pod is launched.", err.Error()) + klog.Errorf("Failed to find created `kube-router` bridge due to %s. Will be created by CNI bridge plugin when pod is launched.", err.Error()) } err = netlink.LinkSetUp(kubeBridgeIf) if err != nil { - glog.Errorf("Failed to bring `kube-router` bridge up due to %s. Will be created by CNI bridge plugin at later point when pod is launched.", err.Error()) + klog.Errorf("Failed to bring `kube-router` bridge up due to %s. Will be created by CNI bridge plugin at later point when pod is launched.", err.Error()) } } if nrc.autoMTU { mtu, err := getMTUFromNodeIP(nrc.nodeIP, nrc.enableOverlays) if err != nil { - glog.Errorf("Failed to find MTU for node IP: %s for intelligently setting the kube-bridge MTU due to %s.", nrc.nodeIP, err.Error()) + klog.Errorf("Failed to find MTU for node IP: %s for intelligently setting the kube-bridge MTU due to %s.", nrc.nodeIP, err.Error()) } if mtu > 0 { - glog.Infof("Setting MTU of kube-bridge interface to: %d", mtu) + klog.Infof("Setting MTU of kube-bridge interface to: %d", mtu) err = netlink.LinkSetMTU(kubeBridgeIf, mtu) if err != nil { - glog.Errorf("Failed to set MTU for kube-bridge interface due to: %s", err.Error()) + klog.Errorf("Failed to set MTU for kube-bridge interface due to: %s", err.Error()) } } else { - glog.Infof("Not setting MTU of kube-bridge interface") + klog.Infof("Not setting MTU of kube-bridge interface") } } // enable netfilter for the bridge if _, err := exec.Command("modprobe", "br_netfilter").CombinedOutput(); err != nil { - glog.Errorf("Failed to enable netfilter for bridge. Network policies and service proxy may not work: %s", err.Error()) + klog.Errorf("Failed to enable netfilter for bridge. Network policies and service proxy may not work: %s", err.Error()) } if err = ioutil.WriteFile("/proc/sys/net/bridge/bridge-nf-call-iptables", []byte(strconv.Itoa(1)), 0640); err != nil { - glog.Errorf("Failed to enable iptables for bridge. Network policies and service proxy may not work: %s", err.Error()) + klog.Errorf("Failed to enable iptables for bridge. Network policies and service proxy may not work: %s", err.Error()) } if nrc.isIpv6 { if err = ioutil.WriteFile("/proc/sys/net/bridge/bridge-nf-call-ip6tables", []byte(strconv.Itoa(1)), 0640); err != nil { - glog.Errorf("Failed to enable ip6tables for bridge. Network policies and service proxy may not work: %s", err.Error()) + klog.Errorf("Failed to enable ip6tables for bridge. Network policies and service proxy may not work: %s", err.Error()) } } @@ -245,19 +245,19 @@ func (nrc *NetworkRoutingController) Run(healthChan chan<- *healthcheck.Controll defer t.Stop() defer wg.Done() - glog.Infof("Starting network route controller") + klog.Infof("Starting network route controller") // Wait till we are ready to launch BGP server for { err := nrc.startBgpServer(true) if err != nil { - glog.Errorf("Failed to start node BGP server: %s", err) + klog.Errorf("Failed to start node BGP server: %s", err) select { case <-stopCh: - glog.Infof("Shutting down network routes controller") + klog.Infof("Shutting down network routes controller") return case <-t.C: - glog.Infof("Retrying start of node BGP server") + klog.Infof("Retrying start of node BGP server") continue } } else { @@ -270,7 +270,7 @@ func (nrc *NetworkRoutingController) Run(healthChan chan<- *healthcheck.Controll defer func() { err := nrc.bgpServer.StopBgp(context.Background(), &gobgpapi.StopBgpRequest{}) if err != nil { - glog.Errorf("error shutting down BGP server: %s", err) + klog.Errorf("error shutting down BGP server: %s", err) } }() } @@ -280,45 +280,45 @@ func (nrc *NetworkRoutingController) Run(healthChan chan<- *healthcheck.Controll var err error select { case <-stopCh: - glog.Infof("Shutting down network routes controller") + klog.Infof("Shutting down network routes controller") return default: } // Update ipset entries if nrc.enablePodEgress || nrc.enableOverlays { - glog.V(1).Info("Syncing ipsets") + klog.V(1).Info("Syncing ipsets") err = nrc.syncNodeIPSets() if err != nil { - glog.Errorf("Error synchronizing ipsets: %s", err.Error()) + klog.Errorf("Error synchronizing ipsets: %s", err.Error()) } } // enable IP forwarding for the packets coming in/out from the pods err = nrc.enableForwarding() if err != nil { - glog.Errorf("Failed to enable IP forwarding of traffic from pods: %s", err.Error()) + klog.Errorf("Failed to enable IP forwarding of traffic from pods: %s", err.Error()) } // advertise or withdraw IPs for the services to be reachable via host toAdvertise, toWithdraw, err := nrc.getActiveVIPs() if err != nil { - glog.Errorf("failed to get routes to advertise/withdraw %s", err) + klog.Errorf("failed to get routes to advertise/withdraw %s", err) } - glog.V(1).Infof("Performing periodic sync of service VIP routes") + klog.V(1).Infof("Performing periodic sync of service VIP routes") nrc.advertiseVIPs(toAdvertise) nrc.withdrawVIPs(toWithdraw) - glog.V(1).Info("Performing periodic sync of pod CIDR routes") + klog.V(1).Info("Performing periodic sync of pod CIDR routes") err = nrc.advertisePodRoute() if err != nil { - glog.Errorf("Error advertising route: %s", err.Error()) + klog.Errorf("Error advertising route: %s", err.Error()) } err = nrc.AddPolicies() if err != nil { - glog.Errorf("Error adding BGP policies: %s", err.Error()) + klog.Errorf("Error adding BGP policies: %s", err.Error()) } if nrc.bgpEnableInternal { @@ -328,13 +328,13 @@ func (nrc *NetworkRoutingController) Run(healthChan chan<- *healthcheck.Controll if err == nil { healthcheck.SendHeartBeat(healthChan, "NRC") } else { - glog.Errorf("Error during periodic sync in network routing controller. Error: " + err.Error()) - glog.Errorf("Skipping sending heartbeat from network routing controller as periodic sync failed.") + klog.Errorf("Error during periodic sync in network routing controller. Error: " + err.Error()) + klog.Errorf("Skipping sending heartbeat from network routing controller as periodic sync failed.") } select { case <-stopCh: - glog.Infof("Shutting down network routes controller") + klog.Infof("Shutting down network routes controller") return case <-t.C: } @@ -344,11 +344,11 @@ func (nrc *NetworkRoutingController) Run(healthChan chan<- *healthcheck.Controll func (nrc *NetworkRoutingController) updateCNIConfig() { cidr, err := utils.GetPodCidrFromCniSpec(nrc.cniConfFile) if err != nil { - glog.Errorf("Failed to get pod CIDR from CNI conf file: %s", err) + klog.Errorf("Failed to get pod CIDR from CNI conf file: %s", err) } if reflect.DeepEqual(cidr, net.IPNet{}) { - glog.Infof("`subnet` in CNI conf file is empty so populating `subnet` in CNI conf file with pod CIDR assigned to the node obtained from node spec.") + klog.Infof("`subnet` in CNI conf file is empty so populating `subnet` in CNI conf file with pod CIDR assigned to the node obtained from node spec.") } cidrlen, _ := cidr.Mask.Size() oldCidr := cidr.IP.String() + "/" + strconv.Itoa(cidrlen) @@ -358,14 +358,14 @@ func (nrc *NetworkRoutingController) updateCNIConfig() { if len(cidr.IP) == 0 || strings.Compare(oldCidr, currentCidr) != 0 { err = utils.InsertPodCidrInCniSpec(nrc.cniConfFile, currentCidr) if err != nil { - glog.Fatalf("Failed to insert `subnet`(pod CIDR) into CNI conf file: %s", err.Error()) + klog.Fatalf("Failed to insert `subnet`(pod CIDR) into CNI conf file: %s", err.Error()) } } if nrc.autoMTU { err = nrc.autoConfigureMTU() if err != nil { - glog.Errorf("Failed to auto-configure MTU due to: %s", err.Error()) + klog.Errorf("Failed to auto-configure MTU due to: %s", err.Error()) } } } @@ -416,9 +416,9 @@ func (nrc *NetworkRoutingController) watchBgpUpdates() { if path.NeighborIp == "" { return } - glog.V(2).Infof("Processing bgp route advertisement from peer: %s", path.NeighborIp) + klog.V(2).Infof("Processing bgp route advertisement from peer: %s", path.NeighborIp) if err := nrc.injectRoute(path); err != nil { - glog.Errorf("Failed to inject routes due to: " + err.Error()) + klog.Errorf("Failed to inject routes due to: " + err.Error()) } } err := nrc.bgpServer.MonitorTable(context.Background(), &gobgpapi.MonitorTableRequest{ @@ -429,7 +429,7 @@ func (nrc *NetworkRoutingController) watchBgpUpdates() { }, }, pathWatch) if err != nil { - glog.Errorf("failed to register monitor global routing table callback due to : " + err.Error()) + klog.Errorf("failed to register monitor global routing table callback due to : " + err.Error()) } } @@ -442,7 +442,7 @@ func (nrc *NetworkRoutingController) advertisePodRoute() error { subnet := cidrStr[0] cidrLen, _ := strconv.Atoi(cidrStr[1]) if nrc.isIpv6 { - glog.V(2).Infof("Advertising route: '%s/%s via %s' to peers", subnet, strconv.Itoa(cidrLen), nrc.nodeIP.String()) + klog.V(2).Infof("Advertising route: '%s/%s via %s' to peers", subnet, strconv.Itoa(cidrLen), nrc.nodeIP.String()) v6Family := &gobgpapi.Family{ Afi: gobgpapi.Family_AFI_IP6, @@ -472,7 +472,7 @@ func (nrc *NetworkRoutingController) advertisePodRoute() error { } } else { - glog.V(2).Infof("Advertising route: '%s/%s via %s' to peers", subnet, strconv.Itoa(cidrLen), nrc.nodeIP.String()) + klog.V(2).Infof("Advertising route: '%s/%s via %s' to peers", subnet, strconv.Itoa(cidrLen), nrc.nodeIP.String()) nlri, _ := ptypes.MarshalAny(&gobgpapi.IPAddressPrefix{ PrefixLen: uint32(cidrLen), Prefix: cidrStr[0], @@ -501,7 +501,7 @@ func (nrc *NetworkRoutingController) advertisePodRoute() error { } func (nrc *NetworkRoutingController) injectRoute(path *gobgpapi.Path) error { - glog.V(2).Infof("injectRoute Path Looks Like: %s", path.String()) + klog.V(2).Infof("injectRoute Path Looks Like: %s", path.String()) var nextHop net.IP out: for _, pAttr := range path.GetPattrs() { @@ -540,24 +540,24 @@ out: // cleanup route and tunnel if overlay is disabled or node is in same subnet and overlay-type is set to 'subnet' if !nrc.enableOverlays || (sameSubnet && nrc.overlayType == "subnet") { - glog.Infof("Cleaning up old routes if there are any") + klog.Infof("Cleaning up old routes if there are any") routes, err := netlink.RouteListFiltered(nl.FAMILY_ALL, &netlink.Route{ Dst: dst, Protocol: 0x11, }, netlink.RT_FILTER_DST|netlink.RT_FILTER_PROTOCOL) if err != nil { - glog.Errorf("Failed to get routes from netlink") + klog.Errorf("Failed to get routes from netlink") } for i, r := range routes { - glog.V(2).Infof("Found route to remove: %s", r.String()) + klog.V(2).Infof("Found route to remove: %s", r.String()) if err := netlink.RouteDel(&routes[i]); err != nil { - glog.Errorf("Failed to remove route due to " + err.Error()) + klog.Errorf("Failed to remove route due to " + err.Error()) } } - glog.Infof("Cleaning up if there is any existing tunnel interface for the node") + klog.Infof("Cleaning up if there is any existing tunnel interface for the node") if link, err := netlink.LinkByName(tunnelName); err == nil { if err = netlink.LinkDel(link); err != nil { - glog.Errorf("Failed to delete tunnel link for the node due to " + err.Error()) + klog.Errorf("Failed to delete tunnel link for the node due to " + err.Error()) } } } @@ -591,7 +591,7 @@ out: return errors.New("Failed to set MTU of tunnel interface " + tunnelName + " up due to: " + err.Error()) } } else { - glog.Infof("Tunnel interface: " + tunnelName + " for the node " + nextHop.String() + " already exists.") + klog.Infof("Tunnel interface: " + tunnelName + " for the node " + nextHop.String() + " already exists.") } out, err := exec.Command("ip", "route", "list", "table", customRouteTableID).CombinedOutput() @@ -619,10 +619,10 @@ out: } if path.IsWithdraw { - glog.V(2).Infof("Removing route: '%s via %s' from peer in the routing table", dst, nextHop) + klog.V(2).Infof("Removing route: '%s via %s' from peer in the routing table", dst, nextHop) return netlink.RouteDel(route) } - glog.V(2).Infof("Inject route: '%s via %s' from peer to routing table", dst, nextHop) + klog.V(2).Infof("Inject route: '%s via %s' from peer to routing table", dst, nextHop) return netlink.RouteReplace(route) } @@ -631,26 +631,26 @@ func (nrc *NetworkRoutingController) Cleanup() { // Pod egress cleanup err := nrc.deletePodEgressRule() if err != nil { - glog.Warningf("Error deleting Pod egress iptables rule: %s", err.Error()) + klog.Warningf("Error deleting Pod egress iptables rule: %s", err.Error()) } err = nrc.deleteBadPodEgressRules() if err != nil { - glog.Warningf("Error deleting Pod egress iptables rule: %s", err.Error()) + klog.Warningf("Error deleting Pod egress iptables rule: %s", err.Error()) } // delete all ipsets created by kube-router ipset, err := utils.NewIPSet(nrc.isIpv6) if err != nil { - glog.Errorf("Failed to clean up ipsets: " + err.Error()) + klog.Errorf("Failed to clean up ipsets: " + err.Error()) } err = ipset.Save() if err != nil { - glog.Errorf("Failed to clean up ipsets: " + err.Error()) + klog.Errorf("Failed to clean up ipsets: " + err.Error()) } err = ipset.DestroyAllWithin() if err != nil { - glog.Warningf("Error deleting ipset: %s", err.Error()) + klog.Warningf("Error deleting ipset: %s", err.Error()) } } @@ -675,13 +675,13 @@ func (nrc *NetworkRoutingController) syncNodeIPSets() error { podCIDR = node.Spec.PodCIDR } if podCIDR == "" { - glog.Warningf("Couldn't determine PodCIDR of the %v node", node.Name) + klog.Warningf("Couldn't determine PodCIDR of the %v node", node.Name) continue } currentPodCidrs = append(currentPodCidrs, podCIDR) nodeIP, err := utils.GetNodeIP(node) if err != nil { - glog.Errorf("Failed to find a node IP, cannot add to node ipset which could affect routing: %v", err) + klog.Errorf("Failed to find a node IP, cannot add to node ipset which could affect routing: %v", err) continue } currentNodeIPs = append(currentNodeIPs, nodeIP.String()) @@ -690,7 +690,7 @@ func (nrc *NetworkRoutingController) syncNodeIPSets() error { // Syncing Pod subnet ipset entries psSet := nrc.ipSetHandler.Get(podSubnetsIPSetName) if psSet == nil { - glog.Infof("Creating missing ipset \"%s\"", podSubnetsIPSetName) + klog.Infof("Creating missing ipset \"%s\"", podSubnetsIPSetName) _, err = nrc.ipSetHandler.Create(podSubnetsIPSetName, utils.OptionTimeout, "0") if err != nil { return fmt.Errorf("ipset \"%s\" not found in controller instance", @@ -705,7 +705,7 @@ func (nrc *NetworkRoutingController) syncNodeIPSets() error { // Syncing Node Addresses ipset entries naSet := nrc.ipSetHandler.Get(nodeAddrsIPSetName) if naSet == nil { - glog.Infof("Creating missing ipset \"%s\"", nodeAddrsIPSetName) + klog.Infof("Creating missing ipset \"%s\"", nodeAddrsIPSetName) _, err = nrc.ipSetHandler.Create(nodeAddrsIPSetName, utils.OptionTimeout, "0") if err != nil { return fmt.Errorf("ipset \"%s\" not found in controller instance", @@ -791,7 +791,7 @@ func (nrc *NetworkRoutingController) startBgpServer(grpcServer bool) error { return errors.New("Could not find ASN number for the node. " + "Node needs to be annotated with ASN number details to start BGP server.") } - glog.Infof("Found ASN for the node to be %s from the node annotations", nodeasn) + klog.Infof("Found ASN for the node to be %s from the node annotations", nodeasn) asnNo, err := strconv.ParseUint(nodeasn, 0, 32) if err != nil { return errors.New("Failed to parse ASN number specified for the the node") @@ -801,7 +801,7 @@ func (nrc *NetworkRoutingController) startBgpServer(grpcServer bool) error { } if clusterid, ok := node.ObjectMeta.Annotations[rrServerAnnotation]; ok { - glog.Infof("Found rr.server for the node to be %s from the node annotation", clusterid) + klog.Infof("Found rr.server for the node to be %s from the node annotation", clusterid) _, err := strconv.ParseUint(clusterid, 0, 32) if err != nil { if ip := net.ParseIP(clusterid).To4(); ip == nil { @@ -811,7 +811,7 @@ func (nrc *NetworkRoutingController) startBgpServer(grpcServer bool) error { nrc.bgpClusterID = clusterid nrc.bgpRRServer = true } else if clusterid, ok := node.ObjectMeta.Annotations[rrClientAnnotation]; ok { - glog.Infof("Found rr.client for the node to be %s from the node annotation", clusterid) + klog.Infof("Found rr.client for the node to be %s from the node annotation", clusterid) _, err := strconv.ParseUint(clusterid, 0, 32) if err != nil { if ip := net.ParseIP(clusterid).To4(); ip == nil { @@ -880,7 +880,7 @@ func (nrc *NetworkRoutingController) startBgpServer(grpcServer bool) error { // Get Global Peer Router ASN configs nodeBgpPeerAsnsAnnotation, ok := node.ObjectMeta.Annotations[peerASNAnnotation] if !ok { - glog.Infof("Could not find BGP peer info for the node in the node annotations so skipping configuring peer.") + klog.Infof("Could not find BGP peer info for the node in the node annotations so skipping configuring peer.") return nil } @@ -889,7 +889,7 @@ func (nrc *NetworkRoutingController) startBgpServer(grpcServer bool) error { if err != nil { err2 := nrc.bgpServer.StopBgp(context.Background(), &gobgpapi.StopBgpRequest{}) if err2 != nil { - glog.Errorf("Failed to stop bgpServer: %s", err2) + klog.Errorf("Failed to stop bgpServer: %s", err2) } return fmt.Errorf("Failed to parse node's Peer ASN Numbers Annotation: %s", err) } @@ -897,7 +897,7 @@ func (nrc *NetworkRoutingController) startBgpServer(grpcServer bool) error { // Get Global Peer Router IP Address configs nodeBgpPeersAnnotation, ok := node.ObjectMeta.Annotations[peerIPAnnotation] if !ok { - glog.Infof("Could not find BGP peer info for the node in the node annotations so skipping configuring peer.") + klog.Infof("Could not find BGP peer info for the node in the node annotations so skipping configuring peer.") return nil } ipStrings := stringToSlice(nodeBgpPeersAnnotation, ",") @@ -905,7 +905,7 @@ func (nrc *NetworkRoutingController) startBgpServer(grpcServer bool) error { if err != nil { err2 := nrc.bgpServer.StopBgp(context.Background(), &gobgpapi.StopBgpRequest{}) if err2 != nil { - glog.Errorf("Failed to stop bgpServer: %s", err2) + klog.Errorf("Failed to stop bgpServer: %s", err2) } return fmt.Errorf("Failed to parse node's Peer Addresses Annotation: %s", err) @@ -921,7 +921,7 @@ func (nrc *NetworkRoutingController) startBgpServer(grpcServer bool) error { if err != nil { err2 := nrc.bgpServer.StopBgp(context.Background(), &gobgpapi.StopBgpRequest{}) if err2 != nil { - glog.Errorf("Failed to stop bgpServer: %s", err2) + klog.Errorf("Failed to stop bgpServer: %s", err2) } return fmt.Errorf("Failed to parse node's Peer Port Numbers Annotation: %s", err) } @@ -931,14 +931,14 @@ func (nrc *NetworkRoutingController) startBgpServer(grpcServer bool) error { var peerPasswords []string nodeBGPPasswordsAnnotation, ok := node.ObjectMeta.Annotations[peerPasswordAnnotation] if !ok { - glog.Infof("Could not find BGP peer password info in the node's annotations. Assuming no passwords.") + klog.Infof("Could not find BGP peer password info in the node's annotations. Assuming no passwords.") } else { passStrings := stringToSlice(nodeBGPPasswordsAnnotation, ",") peerPasswords, err = stringSliceB64Decode(passStrings) if err != nil { err2 := nrc.bgpServer.StopBgp(context.Background(), &gobgpapi.StopBgpRequest{}) if err2 != nil { - glog.Errorf("Failed to stop bgpServer: %s", err2) + klog.Errorf("Failed to stop bgpServer: %s", err2) } return fmt.Errorf("Failed to parse node's Peer Passwords Annotation: %s", err) } @@ -949,7 +949,7 @@ func (nrc *NetworkRoutingController) startBgpServer(grpcServer bool) error { if err != nil { err2 := nrc.bgpServer.StopBgp(context.Background(), &gobgpapi.StopBgpRequest{}) if err2 != nil { - glog.Errorf("Failed to stop bgpServer: %s", err2) + klog.Errorf("Failed to stop bgpServer: %s", err2) } return fmt.Errorf("Failed to process Global Peer Router configs: %s", err) @@ -964,14 +964,14 @@ func (nrc *NetworkRoutingController) startBgpServer(grpcServer bool) error { if err != nil { err2 := nrc.bgpServer.StopBgp(context.Background(), &gobgpapi.StopBgpRequest{}) if err2 != nil { - glog.Errorf("Failed to stop bgpServer: %s", err2) + klog.Errorf("Failed to stop bgpServer: %s", err2) } return fmt.Errorf("Failed to peer with Global Peer Router(s): %s", err) } } else { - glog.Infof("No Global Peer Routers configured. Peering skipped.") + klog.Infof("No Global Peer Routers configured. Peering skipped.") } return nil @@ -1059,7 +1059,7 @@ func NewNetworkRoutingController(clientset kubernetes.Interface, cidr, err := utils.GetPodCidrFromNodeSpec(clientset, nrc.hostnameOverride) if err != nil { - glog.Fatalf("Failed to get pod CIDR from node spec. kube-router relies on kube-controller-manager to allocate pod CIDR for the node or an annotation `kube-router.io/pod-cidr`. Error: %v", err) + klog.Fatalf("Failed to get pod CIDR from node spec. kube-router relies on kube-controller-manager to allocate pod CIDR for the node or an annotation `kube-router.io/pod-cidr`. Error: %v", err) return nil, fmt.Errorf("Failed to get pod CIDR details from Node.spec: %s", err.Error()) } nrc.podCidr = cidr @@ -1149,15 +1149,15 @@ func NewNetworkRoutingController(clientset kubernetes.Interface, bgpLocalAddressListAnnotation, ok := node.ObjectMeta.Annotations[bgpLocalAddressAnnotation] if !ok { - glog.Infof("Could not find annotation `kube-router.io/bgp-local-addresses` on node object so BGP will listen on node IP: %s address.", nrc.nodeIP.String()) + klog.Infof("Could not find annotation `kube-router.io/bgp-local-addresses` on node object so BGP will listen on node IP: %s address.", nrc.nodeIP.String()) nrc.localAddressList = append(nrc.localAddressList, nrc.nodeIP.String()) } else { - glog.Infof("Found annotation `kube-router.io/bgp-local-addresses` on node object so BGP will listen on local IP's: %s", bgpLocalAddressListAnnotation) + klog.Infof("Found annotation `kube-router.io/bgp-local-addresses` on node object so BGP will listen on local IP's: %s", bgpLocalAddressListAnnotation) localAddresses := stringToSlice(bgpLocalAddressListAnnotation, ",") for _, addr := range localAddresses { ip := net.ParseIP(addr) if ip == nil { - glog.Fatalf("Invalid IP address %s specified in `kube-router.io/bgp-local-addresses`.", addr) + klog.Fatalf("Invalid IP address %s specified in `kube-router.io/bgp-local-addresses`.", addr) } } nrc.localAddressList = append(nrc.localAddressList, localAddresses...) diff --git a/pkg/controllers/routing/pod_egress.go b/pkg/controllers/routing/pod_egress.go index 1fa1e261a5..5b3c10fd39 100644 --- a/pkg/controllers/routing/pod_egress.go +++ b/pkg/controllers/routing/pod_egress.go @@ -4,7 +4,7 @@ import ( "errors" "fmt" - "github.com/golang/glog" + "k8s.io/klog/v2" ) // set up MASQUERADE rule so that egress traffic from the pods gets masqueraded to node's IP @@ -47,7 +47,7 @@ func (nrc *NetworkRoutingController) createPodEgressRule() error { } - glog.V(1).Infof("Added iptables rule to masquerade outbound traffic from pods.") + klog.V(1).Infof("Added iptables rule to masquerade outbound traffic from pods.") return nil } @@ -76,7 +76,7 @@ func (nrc *NetworkRoutingController) deletePodEgressRule() error { return errors.New("Failed to delete iptables rule to masquerade outbound traffic from pods: " + err.Error() + ". Pod egress might still work...") } - glog.Infof("Deleted iptables rule to masquerade outbound traffic from pods.") + klog.Infof("Deleted iptables rule to masquerade outbound traffic from pods.") } return nil @@ -115,7 +115,7 @@ func (nrc *NetworkRoutingController) deleteBadPodEgressRules() error { "Pod egress might still work, or bugs may persist after upgrade...", err) } - glog.Infof("Deleted old/bad iptables rule to masquerade outbound traffic from pods.") + klog.Infof("Deleted old/bad iptables rule to masquerade outbound traffic from pods.") } } diff --git a/pkg/cri/remote_runtime.go b/pkg/cri/remote_runtime.go index 6c1c20e1fe..21f2bca2fd 100644 --- a/pkg/cri/remote_runtime.go +++ b/pkg/cri/remote_runtime.go @@ -8,8 +8,8 @@ import ( "strings" "time" - "github.com/golang/glog" "google.golang.org/grpc" + "k8s.io/klog/v2" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -37,7 +37,7 @@ func NewRemoteRuntimeService(endpoint string, connectionTimeout time.Duration) ( return nil, err } - glog.V(4).Infof("[RuntimeService] got endpoint %s (proto=%s, path=%s)", endpoint, proto, addr) + klog.V(4).Infof("[RuntimeService] got endpoint %s (proto=%s, path=%s)", endpoint, proto, addr) if proto != "unix" { return nil, errors.New("[RuntimeService] only unix socket is currently supported") @@ -48,7 +48,7 @@ func NewRemoteRuntimeService(endpoint string, connectionTimeout time.Duration) ( conn, err := grpc.DialContext(ctx, addr, grpc.WithInsecure(), grpc.WithContextDialer(dialer), grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize))) if err != nil { - glog.Errorf("Connect remote runtime %s failed: %v", addr, err) + klog.Errorf("Connect remote runtime %s failed: %v", addr, err) return nil, err } diff --git a/pkg/healthcheck/health_controller.go b/pkg/healthcheck/health_controller.go index 89d819203c..40bca2fdb8 100644 --- a/pkg/healthcheck/health_controller.go +++ b/pkg/healthcheck/health_controller.go @@ -7,8 +7,8 @@ import ( "time" "github.com/cloudnativelabs/kube-router/pkg/options" - "github.com/golang/glog" "golang.org/x/net/context" + "k8s.io/klog/v2" ) //ControllerHeartbeat is the structure to hold the heartbeats sent by controllers @@ -53,7 +53,7 @@ func (hc *HealthController) Handler(w http.ResponseWriter, req *http.Request) { w.WriteHeader(http.StatusOK) _, err := w.Write([]byte("OK\n")) if err != nil { - glog.Errorf("Failed to write body: %s", err) + klog.Errorf("Failed to write body: %s", err) } } else { w.WriteHeader(http.StatusInternalServerError) @@ -70,14 +70,14 @@ func (hc *HealthController) Handler(w http.ResponseWriter, req *http.Request) { */ _, err := w.Write([]byte("Unhealthy")) if err != nil { - glog.Errorf("Failed to write body: %s", err) + klog.Errorf("Failed to write body: %s", err) } } } //HandleHeartbeat handles received heartbeats on the health channel func (hc *HealthController) HandleHeartbeat(beat *ControllerHeartbeat) { - glog.V(3).Infof("Received heartbeat from %s", beat.Component) + klog.V(3).Infof("Received heartbeat from %s", beat.Component) hc.Status.Lock() defer hc.Status.Unlock() @@ -114,28 +114,28 @@ func (hc *HealthController) CheckHealth() bool { if hc.Config.RunFirewall { if time.Since(hc.Status.NetworkPolicyControllerAlive) > hc.Config.IPTablesSyncPeriod+hc.Status.NetworkPolicyControllerAliveTTL+graceTime { - glog.Error("Network Policy Controller heartbeat missed") + klog.Error("Network Policy Controller heartbeat missed") health = false } } if hc.Config.RunRouter { if time.Since(hc.Status.NetworkRoutingControllerAlive) > hc.Config.RoutesSyncPeriod+hc.Status.NetworkRoutingControllerAliveTTL+graceTime { - glog.Error("Network Routing Controller heartbeat missed") + klog.Error("Network Routing Controller heartbeat missed") health = false } } if hc.Config.RunServiceProxy { if time.Since(hc.Status.NetworkServicesControllerAlive) > hc.Config.IpvsSyncPeriod+hc.Status.NetworkServicesControllerAliveTTL+graceTime { - glog.Error("NetworkService Controller heartbeat missed") + klog.Error("NetworkService Controller heartbeat missed") health = false } } if hc.Config.MetricsEnabled { if time.Since(hc.Status.MetricsControllerAlive) > 5*time.Second { - glog.Error("Metrics Controller heartbeat missed") + klog.Error("Metrics Controller heartbeat missed") health = false } } @@ -153,21 +153,21 @@ func (hc *HealthController) RunServer(stopCh <-chan struct{}, wg *sync.WaitGroup go func() { if err := srv.ListenAndServe(); err != nil { // cannot panic, because this probably is an intentional close - glog.Errorf("Health controller error: %s", err) + klog.Errorf("Health controller error: %s", err) } }() } else if hc.Config.MetricsPort > 65535 { - glog.Errorf("Metrics port must be over 0 and under 65535, given port: %d", hc.Config.MetricsPort) + klog.Errorf("Metrics port must be over 0 and under 65535, given port: %d", hc.Config.MetricsPort) } else { hc.HTTPEnabled = false } // block until we receive a shut down signal <-stopCh - glog.Infof("Shutting down health controller") + klog.Infof("Shutting down health controller") if hc.HTTPEnabled { if err := srv.Shutdown(context.Background()); err != nil { - glog.Errorf("could not shutdown: %v", err) + klog.Errorf("could not shutdown: %v", err) } } } @@ -179,12 +179,12 @@ func (hc *HealthController) RunCheck(healthChan <-chan *ControllerHeartbeat, sto for { select { case <-stopCh: - glog.Infof("Shutting down HealthController RunCheck") + klog.Infof("Shutting down HealthController RunCheck") return case heartbeat := <-healthChan: hc.HandleHeartbeat(heartbeat) case <-t.C: - glog.V(4).Info("Health controller tick") + klog.V(4).Info("Health controller tick") } hc.Status.Healthy = hc.CheckHealth() } diff --git a/pkg/metrics/metrics_controller.go b/pkg/metrics/metrics_controller.go index a6a414e203..4f49ccb05f 100644 --- a/pkg/metrics/metrics_controller.go +++ b/pkg/metrics/metrics_controller.go @@ -10,11 +10,11 @@ import ( "github.com/cloudnativelabs/kube-router/pkg/healthcheck" "github.com/cloudnativelabs/kube-router/pkg/options" "github.com/cloudnativelabs/kube-router/pkg/version" - "github.com/golang/glog" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "golang.org/x/net/context" "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" ) const ( @@ -172,7 +172,7 @@ type Controller struct { func (mc *Controller) Run(healthChan chan<- *healthcheck.ControllerHeartbeat, stopCh <-chan struct{}, wg *sync.WaitGroup) { t := time.NewTicker(3 * time.Second) defer wg.Done() - glog.Info("Starting metrics controller") + klog.Info("Starting metrics controller") // register metrics for this controller BuildInfo.WithLabelValues(runtime.Version(), version.Version).Set(1) @@ -187,20 +187,20 @@ func (mc *Controller) Run(healthChan chan<- *healthcheck.ControllerHeartbeat, st go func() { if err := srv.ListenAndServe(); err != nil { // cannot panic, because this probably is an intentional close - glog.Errorf("Metrics controller error: %s", err) + klog.Errorf("Metrics controller error: %s", err) } }() for { healthcheck.SendHeartBeat(healthChan, "MC") select { case <-stopCh: - glog.Infof("Shutting down metrics controller") + klog.Infof("Shutting down metrics controller") if err := srv.Shutdown(context.Background()); err != nil { - glog.Errorf("could not shutdown: %v", err) + klog.Errorf("could not shutdown: %v", err) } return case <-t.C: - glog.V(4).Info("Metrics controller tick") + klog.V(4).Info("Metrics controller tick") } } } diff --git a/pkg/version/version.go b/pkg/version/version.go index 63c867ffef..482bb821a5 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -5,7 +5,7 @@ import ( "os" "runtime" - "github.com/golang/glog" + "k8s.io/klog/v2" ) // Version and BuildDate are injected at build time via ldflags @@ -18,6 +18,6 @@ func PrintVersion(logOutput bool) { if !logOutput { fmt.Fprintf(os.Stderr, "%s", output) } else { - glog.Info(output) + klog.Info(output) } }