From 925bb1a5168655f0ca493c8292a25fe730fc1e83 Mon Sep 17 00:00:00 2001 From: Alex Jones Date: Tue, 9 May 2023 20:03:44 +0100 Subject: [PATCH 1/3] chore: guard against missing secrets Signed-off-by: Alex Jones --- pkg/resources/k8sgpt.go | 43 +++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/pkg/resources/k8sgpt.go b/pkg/resources/k8sgpt.go index ec2a9945..fb962211 100644 --- a/pkg/resources/k8sgpt.go +++ b/pkg/resources/k8sgpt.go @@ -25,6 +25,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -253,37 +254,37 @@ func Sync(ctx context.Context, c client.Client, var objs []client.Object - svc, err := GetService(config) - if err != nil { - return err + svc, er := GetService(config) + if er != nil { + return er } objs = append(objs, svc) - svcAcc, err := GetServiceAccount(config) - if err != nil { - return err + svcAcc, er := GetServiceAccount(config) + if er != nil { + return er } objs = append(objs, svcAcc) - clusterRole, err := GetClusterRole(config) - if err != nil { - return err + clusterRole, er := GetClusterRole(config) + if er != nil { + return er } objs = append(objs, clusterRole) - clusterRoleBinding, err := GetClusterRoleBinding(config) - if err != nil { - return err + clusterRoleBinding, er := GetClusterRoleBinding(config) + if er != nil { + return er } objs = append(objs, clusterRoleBinding) - deployment, err := GetDeployment(config) - if err != nil { - return err + deployment, er := GetDeployment(config) + if er != nil { + return er } objs = append(objs, deployment) @@ -292,6 +293,18 @@ func Sync(ctx context.Context, c client.Client, for _, obj := range objs { switch i { case Create: + + // before creation we will check to see if the secret exists if used as a ref + if config.Spec.Secret != nil { + + secret := &v1.Secret{} + er := c.Get(ctx, types.NamespacedName{Name: config.Spec.Secret.Name, + Namespace: config.Namespace}, secret) + if er != nil { + return err.New("references secret does not exist, cannot create deployment") + } + } + err := c.Create(ctx, obj) if err != nil { // If the object already exists, ignore the error From dd6da99569cc2cd1b039fe914b231b8759986d2a Mon Sep 17 00:00:00 2001 From: Alex Jones Date: Wed, 10 May 2023 11:57:39 +0100 Subject: [PATCH 2/3] fix: use direct pod IP to bypass DNS caching issues Signed-off-by: Alex Jones --- controllers/k8sgpt_controller.go | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/controllers/k8sgpt_controller.go b/controllers/k8sgpt_controller.go index 853e7b3b..fddd350f 100644 --- a/controllers/k8sgpt_controller.go +++ b/controllers/k8sgpt_controller.go @@ -17,17 +17,18 @@ package controllers import ( "context" "fmt" + "net" "os" "strings" "time" corev1alpha1 "github.com/k8sgpt-ai/k8sgpt-operator/api/v1alpha1" - kclient "github.com/k8sgpt-ai/k8sgpt-operator/pkg/client" "github.com/k8sgpt-ai/k8sgpt-operator/pkg/resources" "github.com/k8sgpt-ai/k8sgpt-operator/pkg/utils" "github.com/prometheus/client_golang/prometheus" v1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -144,9 +145,36 @@ func (r *K8sGPTReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr if os.Getenv("LOCAL_MODE") != "" { address = "localhost:8080" } else { - address = fmt.Sprintf("%s.%s:8080", "k8sgpt", deployment.Namespace) + // Get k8sgpt-deployment service pod ip + podList := &corev1.PodList{} + listOpts := []client.ListOption{ + client.InNamespace(k8sgptConfig.Namespace), + client.MatchingLabels{"app": "k8sgpt-deployment"}, + } + err := r.List(ctx, podList, listOpts...) + if err != nil { + k8sgptReconcileErrorCount.Inc() + return r.finishReconcile(err, false) + } + if len(podList.Items) == 0 { + k8sgptReconcileErrorCount.Inc() + return r.finishReconcile(fmt.Errorf("no pods found for k8sgpt-deployment"), false) + } + address = fmt.Sprintf("%s:8080", podList.Items[0].Status.PodIP) } + fmt.Printf("Creating new client for %s\n", address) + // Test if the port is open + conn, err := net.DialTimeout("tcp", address, 1*time.Second) + if err != nil { + k8sgptReconcileErrorCount.Inc() + return r.finishReconcile(err, false) + } + + fmt.Printf("Connection established between %s and localhost with time out of %d seconds.\n", address, int64(1)) + fmt.Printf("Remote Address : %s \n", conn.RemoteAddr().String()) + fmt.Printf("Local Address : %s \n", conn.LocalAddr().String()) + k8sgptClient, err := kclient.NewClient(address) if err != nil { k8sgptReconcileErrorCount.Inc() From ed4173db2fe6f9938cf37245bc44b0c793c2332a Mon Sep 17 00:00:00 2001 From: Alex Jones Date: Wed, 10 May 2023 12:09:52 +0100 Subject: [PATCH 3/3] chore: solving the issue with reconciliation Signed-off-by: Alex Jones --- .gitignore | 1 + controllers/k8sgpt_controller.go | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 56725f27..02302d83 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +__debug_bin *.DS_Store k8sgpt-operator # Binaries for programs and plugins diff --git a/controllers/k8sgpt_controller.go b/controllers/k8sgpt_controller.go index fddd350f..5af757df 100644 --- a/controllers/k8sgpt_controller.go +++ b/controllers/k8sgpt_controller.go @@ -212,8 +212,16 @@ func (r *K8sGPTReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr // if the result already exists, we will update it if errors.IsAlreadyExists(err) { - result.ResourceVersion = k8sgptConfig.GetResourceVersion() - err = r.Update(ctx, &result) + // Get the actual result with metadata rather than our local construct + var newResult corev1alpha1.Result + err = r.Get(ctx, client.ObjectKey{Namespace: k8sgptConfig.Namespace, + Name: name}, &newResult) + if err != nil { + k8sgptReconcileErrorCount.Inc() + return r.finishReconcile(err, false) + } + newResult.Spec = resultSpec + err = r.Update(ctx, &newResult) if err != nil { k8sgptReconcileErrorCount.Inc() return r.finishReconcile(err, false)