Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Avoid setting owner refs if the service mesh/ingress is on a different cluster #1183

Merged
merged 1 commit into from
May 6, 2022
Merged

Avoid setting owner refs if the service mesh/ingress is on a different cluster #1183

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 12 additions & 5 deletions cmd/flagger/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,15 +168,15 @@ func main() {
if kubeconfigServiceMesh == "" {
kubeconfigServiceMesh = kubeconfig
}
cfgHost, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfigServiceMesh)
serviceMeshCfg, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfigServiceMesh)
if err != nil {
logger.Fatalf("Error building host kubeconfig: %v", err)
}

cfgHost.QPS = float32(kubeconfigQPS)
cfgHost.Burst = kubeconfigBurst
serviceMeshCfg.QPS = float32(kubeconfigQPS)
serviceMeshCfg.Burst = kubeconfigBurst

meshClient, err := clientset.NewForConfig(cfgHost)
meshClient, err := clientset.NewForConfig(serviceMeshCfg)
if err != nil {
logger.Fatalf("Error building mesh clientset: %v", err)
}
Expand Down Expand Up @@ -212,7 +212,14 @@ func main() {
// start HTTP server
go server.ListenAndServe(port, 3*time.Second, logger, stopCh)

routerFactory := router.NewFactory(cfg, kubeClient, flaggerClient, ingressAnnotationsPrefix, ingressClass, logger, meshClient)
setOwnerRefs := true
// Router shouldn't set OwnerRefs on resources that they create since the
// service mesh/ingress controller is in a different cluster.
if cfg.Host != serviceMeshCfg.Host {
setOwnerRefs = false
}

routerFactory := router.NewFactory(cfg, kubeClient, flaggerClient, ingressAnnotationsPrefix, ingressClass, logger, meshClient, setOwnerRefs)

var configTracker canary.Tracker
if enableConfigTracking {
Expand Down
2 changes: 1 addition & 1 deletion pkg/controller/scheduler_daemonset_fixture_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ func newDaemonSetFixture(c *flaggerv1.Canary) daemonSetFixture {
}

// init router
rf := router.NewFactory(nil, kubeClient, flaggerClient, "annotationsPrefix", "", logger, flaggerClient)
rf := router.NewFactory(nil, kubeClient, flaggerClient, "annotationsPrefix", "", logger, flaggerClient, true)

// init observer
observerFactory, _ := observers.NewFactory(testMetricsServerURL)
Expand Down
2 changes: 1 addition & 1 deletion pkg/controller/scheduler_deployment_fixture_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ func newDeploymentFixture(c *flaggerv1.Canary) fixture {
}

// init router
rf := router.NewFactory(nil, kubeClient, flaggerClient, "annotationsPrefix", "", logger, flaggerClient)
rf := router.NewFactory(nil, kubeClient, flaggerClient, "annotationsPrefix", "", logger, flaggerClient, true)

// init observer
observerFactory, _ := observers.NewFactory(testMetricsServerURL)
Expand Down
33 changes: 19 additions & 14 deletions pkg/router/appmesh.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ type AppMeshRouter struct {
appmeshClient clientset.Interface
flaggerClient clientset.Interface
logger *zap.SugaredLogger
setOwnerRefs bool
}

// Reconcile creates or updates App Mesh virtual nodes and virtual services
Expand Down Expand Up @@ -161,16 +162,18 @@ func (ar *AppMeshRouter) reconcileVirtualNode(canary *flaggerv1.Canary, name str
Namespace: canary.Namespace,
Labels: metadata.Labels,
Annotations: filterMetadata(metadata.Annotations),
OwnerReferences: []metav1.OwnerReference{
*metav1.NewControllerRef(canary, schema.GroupVersionKind{
Group: flaggerv1.SchemeGroupVersion.Group,
Version: flaggerv1.SchemeGroupVersion.Version,
Kind: flaggerv1.CanaryKind,
}),
},
},
Spec: vnSpec,
}
if ar.setOwnerRefs {
virtualnode.OwnerReferences = []metav1.OwnerReference{
*metav1.NewControllerRef(canary, schema.GroupVersionKind{
Group: flaggerv1.SchemeGroupVersion.Group,
Version: flaggerv1.SchemeGroupVersion.Version,
Kind: flaggerv1.CanaryKind,
}),
}
}
_, err = ar.appmeshClient.AppmeshV1beta1().VirtualNodes(canary.Namespace).Create(context.TODO(), virtualnode, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("VirtualNode %s.%s create error %w", name, canary.Namespace, err)
Expand Down Expand Up @@ -314,16 +317,18 @@ func (ar *AppMeshRouter) reconcileVirtualService(canary *flaggerv1.Canary, name
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: canary.Namespace,
OwnerReferences: []metav1.OwnerReference{
*metav1.NewControllerRef(canary, schema.GroupVersionKind{
Group: flaggerv1.SchemeGroupVersion.Group,
Version: flaggerv1.SchemeGroupVersion.Version,
Kind: flaggerv1.CanaryKind,
}),
},
},
Spec: vsSpec,
}
if ar.setOwnerRefs {
virtualService.OwnerReferences = []metav1.OwnerReference{
*metav1.NewControllerRef(canary, schema.GroupVersionKind{
Group: flaggerv1.SchemeGroupVersion.Group,
Version: flaggerv1.SchemeGroupVersion.Version,
Kind: flaggerv1.CanaryKind,
}),
}
}

// set App Mesh Gateway annotation on primary virtual service
if canaryWeight == 0 {
Expand Down
49 changes: 28 additions & 21 deletions pkg/router/appmesh_v1beta2.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ type AppMeshv1beta2Router struct {
flaggerClient clientset.Interface
logger *zap.SugaredLogger
labelSelector string
setOwnerRefs bool
}

// Reconcile creates or updates App Mesh virtual nodes and virtual services
Expand Down Expand Up @@ -175,16 +176,18 @@ func (ar *AppMeshv1beta2Router) reconcileVirtualNode(canary *flaggerv1.Canary, n
Namespace: canary.Namespace,
Labels: metadata.Labels,
Annotations: filterMetadata(metadata.Annotations),
OwnerReferences: []metav1.OwnerReference{
*metav1.NewControllerRef(canary, schema.GroupVersionKind{
Group: flaggerv1.SchemeGroupVersion.Group,
Version: flaggerv1.SchemeGroupVersion.Version,
Kind: flaggerv1.CanaryKind,
}),
},
},
Spec: vnSpec,
}
if ar.setOwnerRefs {
virtualnode.OwnerReferences = []metav1.OwnerReference{
*metav1.NewControllerRef(canary, schema.GroupVersionKind{
Group: flaggerv1.SchemeGroupVersion.Group,
Version: flaggerv1.SchemeGroupVersion.Version,
Kind: flaggerv1.CanaryKind,
}),
}
}
_, err = ar.appmeshClient.AppmeshV1beta2().VirtualNodes(canary.Namespace).Create(context.TODO(), virtualnode, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("VirtualNode %s.%s create error %w", name, canary.Namespace, err)
Expand Down Expand Up @@ -341,16 +344,18 @@ func (ar *AppMeshv1beta2Router) reconcileVirtualRouter(canary *flaggerv1.Canary,
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: canary.Namespace,
OwnerReferences: []metav1.OwnerReference{
*metav1.NewControllerRef(canary, schema.GroupVersionKind{
Group: flaggerv1.SchemeGroupVersion.Group,
Version: flaggerv1.SchemeGroupVersion.Version,
Kind: flaggerv1.CanaryKind,
}),
},
},
Spec: vrSpec,
}
if ar.setOwnerRefs {
virtualRouter.OwnerReferences = []metav1.OwnerReference{
*metav1.NewControllerRef(canary, schema.GroupVersionKind{
Group: flaggerv1.SchemeGroupVersion.Group,
Version: flaggerv1.SchemeGroupVersion.Version,
Kind: flaggerv1.CanaryKind,
}),
}
}

_, err = ar.appmeshClient.AppmeshV1beta2().VirtualRouters(canary.Namespace).Create(context.TODO(), virtualRouter, metav1.CreateOptions{})
if err != nil {
Expand All @@ -363,13 +368,6 @@ func (ar *AppMeshv1beta2Router) reconcileVirtualRouter(canary *flaggerv1.Canary,
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: canary.Namespace,
OwnerReferences: []metav1.OwnerReference{
*metav1.NewControllerRef(canary, schema.GroupVersionKind{
Group: flaggerv1.SchemeGroupVersion.Group,
Version: flaggerv1.SchemeGroupVersion.Version,
Kind: flaggerv1.CanaryKind,
}),
},
},
Spec: appmeshv1.VirtualServiceSpec{
Provider: &appmeshv1.VirtualServiceProvider{
Expand All @@ -381,6 +379,15 @@ func (ar *AppMeshv1beta2Router) reconcileVirtualRouter(canary *flaggerv1.Canary,
},
},
}
if ar.setOwnerRefs {
virtualService.OwnerReferences = []metav1.OwnerReference{
*metav1.NewControllerRef(canary, schema.GroupVersionKind{
Group: flaggerv1.SchemeGroupVersion.Group,
Version: flaggerv1.SchemeGroupVersion.Version,
Kind: flaggerv1.CanaryKind,
}),
}
}

// set App Mesh Gateway annotation on primary virtual service
if canaryWeight == 0 {
Expand Down
17 changes: 10 additions & 7 deletions pkg/router/contour.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ type ContourRouter struct {
flaggerClient clientset.Interface
logger *zap.SugaredLogger
ingressClass string
setOwnerRefs bool
}

// Reconcile creates or updates the HTTP proxy
Expand Down Expand Up @@ -169,20 +170,22 @@ func (cr *ContourRouter) Reconcile(canary *flaggerv1.Canary) error {
Namespace: canary.Namespace,
Labels: metadata.Labels,
Annotations: filterMetadata(metadata.Annotations),
OwnerReferences: []metav1.OwnerReference{
*metav1.NewControllerRef(canary, schema.GroupVersionKind{
Group: flaggerv1.SchemeGroupVersion.Group,
Version: flaggerv1.SchemeGroupVersion.Version,
Kind: flaggerv1.CanaryKind,
}),
},
},
Spec: newSpec,
Status: contourv1.HTTPProxyStatus{
CurrentStatus: "valid",
Description: "valid HTTPProxy",
},
}
if cr.setOwnerRefs {
proxy.OwnerReferences = []metav1.OwnerReference{
*metav1.NewControllerRef(canary, schema.GroupVersionKind{
Group: flaggerv1.SchemeGroupVersion.Group,
Version: flaggerv1.SchemeGroupVersion.Version,
Kind: flaggerv1.CanaryKind,
}),
}
}

if cr.ingressClass != "" {
proxy.Annotations = map[string]string{
Expand Down
24 changes: 21 additions & 3 deletions pkg/router/factory.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,14 +35,16 @@ type Factory struct {
ingressAnnotationsPrefix string
ingressClass string
logger *zap.SugaredLogger
setOwnerRefs bool
}

func NewFactory(kubeConfig *restclient.Config, kubeClient kubernetes.Interface,
flaggerClient clientset.Interface,
ingressAnnotationsPrefix string,
ingressClass string,
logger *zap.SugaredLogger,
meshClient clientset.Interface) *Factory {
meshClient clientset.Interface,
setOwnerRefs bool) *Factory {
return &Factory{
kubeConfig: kubeConfig,
meshClient: meshClient,
Expand All @@ -51,6 +53,7 @@ func NewFactory(kubeConfig *restclient.Config, kubeClient kubernetes.Interface,
ingressAnnotationsPrefix: ingressAnnotationsPrefix,
ingressClass: ingressClass,
logger: logger,
setOwnerRefs: setOwnerRefs,
}
}

Expand Down Expand Up @@ -81,13 +84,15 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf
kubeClient: factory.kubeClient,
appmeshClient: factory.meshClient,
labelSelector: labelSelector,
setOwnerRefs: factory.setOwnerRefs,
}
case provider == flaggerv1.AppMeshProvider:
return &AppMeshRouter{
logger: factory.logger,
flaggerClient: factory.flaggerClient,
kubeClient: factory.kubeClient,
appmeshClient: factory.meshClient,
setOwnerRefs: factory.setOwnerRefs,
}
case provider == flaggerv1.LinkerdProvider:
return &SmiRouter{
Expand All @@ -96,13 +101,15 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf
kubeClient: factory.kubeClient,
smiClient: factory.meshClient,
targetMesh: flaggerv1.LinkerdProvider,
setOwnerRefs: factory.setOwnerRefs,
}
case provider == flaggerv1.IstioProvider:
return &IstioRouter{
logger: factory.logger,
flaggerClient: factory.flaggerClient,
kubeClient: factory.kubeClient,
istioClient: factory.meshClient,
setOwnerRefs: factory.setOwnerRefs,
}
case strings.HasPrefix(provider, flaggerv1.SMIProvider+":v1alpha1"):
mesh := strings.TrimPrefix(provider, flaggerv1.SMIProvider+":v1alpha1:")
Expand All @@ -112,6 +119,7 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf
kubeClient: factory.kubeClient,
smiClient: factory.meshClient,
targetMesh: mesh,
setOwnerRefs: factory.setOwnerRefs,
}
case strings.HasPrefix(provider, flaggerv1.SMIProvider+":v1alpha2"):
mesh := strings.TrimPrefix(provider, flaggerv1.SMIProvider+":v1alpha2:")
Expand All @@ -121,6 +129,7 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf
kubeClient: factory.kubeClient,
smiClient: factory.meshClient,
targetMesh: mesh,
setOwnerRefs: factory.setOwnerRefs,
}
case strings.HasPrefix(provider, flaggerv1.SMIProvider+":v1alpha3"):
mesh := strings.TrimPrefix(provider, flaggerv1.SMIProvider+":v1alpha3:")
Expand All @@ -130,6 +139,7 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf
kubeClient: factory.kubeClient,
smiClient: factory.meshClient,
targetMesh: mesh,
setOwnerRefs: factory.setOwnerRefs,
}
case provider == flaggerv1.ContourProvider:
return &ContourRouter{
Expand All @@ -138,29 +148,34 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf
kubeClient: factory.kubeClient,
contourClient: factory.meshClient,
ingressClass: factory.ingressClass,
setOwnerRefs: factory.setOwnerRefs,
}
case strings.HasPrefix(provider, flaggerv1.GlooProvider):
return &GlooRouter{
logger: factory.logger,
flaggerClient: factory.flaggerClient,
kubeClient: factory.kubeClient,
glooClient: factory.meshClient,
setOwnerRefs: factory.setOwnerRefs,
}
case provider == flaggerv1.NGINXProvider:
return &IngressRouter{
logger: factory.logger,
kubeClient: factory.kubeClient,
annotationsPrefix: factory.ingressAnnotationsPrefix,
setOwnerRefs: factory.setOwnerRefs,
}
case provider == flaggerv1.SkipperProvider:
return &SkipperRouter{
logger: factory.logger,
kubeClient: factory.kubeClient,
logger: factory.logger,
kubeClient: factory.kubeClient,
setOwnerRefs: factory.setOwnerRefs,
}
case provider == flaggerv1.TraefikProvider:
return &TraefikRouter{
logger: factory.logger,
traefikClient: factory.meshClient,
setOwnerRefs: factory.setOwnerRefs,
}
case provider == flaggerv1.OsmProvider:
return &Smiv1alpha2Router{
Expand All @@ -169,6 +184,7 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf
kubeClient: factory.kubeClient,
smiClient: factory.meshClient,
targetMesh: flaggerv1.OsmProvider,
setOwnerRefs: factory.setOwnerRefs,
}
case provider == flaggerv1.KumaProvider:
return &KumaRouter{
Expand All @@ -182,6 +198,7 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf
logger: factory.logger,
kubeClient: factory.kubeClient,
gatewayAPIClient: factory.meshClient,
setOwnerRefs: factory.setOwnerRefs,
}
case provider == flaggerv1.KubernetesProvider:
return &NopRouter{}
Expand All @@ -191,6 +208,7 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf
flaggerClient: factory.flaggerClient,
kubeClient: factory.kubeClient,
istioClient: factory.meshClient,
setOwnerRefs: factory.setOwnerRefs,
}
}
}
Loading