diff --git a/k8s/controllers/k8s/sidecar_controller.go b/k8s/controllers/k8s/sidecar_controller.go index a6b5762d..1a52d28c 100644 --- a/k8s/controllers/k8s/sidecar_controller.go +++ b/k8s/controllers/k8s/sidecar_controller.go @@ -77,11 +77,11 @@ func (r *SidecarReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct etcdCfg.Rank = rank return etcdCfg } - if _, err := sidecarApp.Apply(ctx, "etcd/etcd.yaml", logger, false); err != nil { + if _, err := sidecarApp.Apply(ctx, "etcd/etcd.yaml", logger, true); err != nil { logger.Error(err, "failed to apply etcd pod") return ctrl.Result{}, err } - if _, err := sidecarApp.Apply(ctx, "etcd/service.yaml", logger, false); err != nil { + if _, err := sidecarApp.Apply(ctx, "etcd/service.yaml", logger, true); err != nil { logger.Error(err, "failed to apply etcd service") return ctrl.Result{}, err } diff --git a/k8s/pkg/schedulers/common.go b/k8s/pkg/schedulers/common.go index 75b27b0b..4ed2489a 100644 --- a/k8s/pkg/schedulers/common.go +++ b/k8s/pkg/schedulers/common.go @@ -69,6 +69,9 @@ func GetRequiredJob(anno map[string]string) []string { if !exists { return []string{} } + if requiredJobs == "" { + return []string{} + } return strings.Split(requiredJobs, ",") } diff --git a/k8s/pkg/webhook/sidecar/sidecar_webhook.go b/k8s/pkg/webhook/sidecar/sidecar_webhook.go index 278cdeab..c3fc0a57 100644 --- a/k8s/pkg/webhook/sidecar/sidecar_webhook.go +++ b/k8s/pkg/webhook/sidecar/sidecar_webhook.go @@ -22,6 +22,7 @@ import ( "encoding/json" "html/template" "net/http" + "regexp" "sort" "strings" @@ -33,6 +34,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -84,9 +86,15 @@ func (r *Injector) Handle(ctx context.Context, req admission.Request) admission. sort.Strings(keys) selectorname := strings.Join([]string{keys[0], l[keys[0]]}, "-") + if len(selectorname) > 63-len("-default-sidecar") { + selectorname = selectorname[:63-len("-default-sidecar")] + } sidecar.Name = selectorname + "-default-sidecar" + // replace the invalid characters in the name + re := regexp.MustCompile(`[^a-zA-Z0-9.-]+`) + sidecar.Name = re.ReplaceAllString(sidecar.Name, "-") sidecar.Namespace = pod.Namespace - + sidecar.OwnerReferences = pod.OwnerReferences err := r.Get( ctx, types.NamespacedName{Name: sidecar.Name, Namespace: sidecar.Namespace}, @@ -107,9 +115,18 @@ func (r *Injector) Handle(ctx context.Context, req admission.Request) admission. return admission.Errored(http.StatusInternalServerError, err) } } else { - // the default sidecar cr exists, update it - sidecar.Spec.Replicas++ - if err := r.Update(ctx, sidecar); err != nil { + if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + name := client.ObjectKeyFromObject(sidecar) + if err := r.Get(ctx, name, sidecar); err != nil { + return errors.Wrap(err, "failed to get "+name.String()) + } + // the default sidecar cr exists, update it + sidecar.Spec.Replicas++ + if err := r.Update(ctx, sidecar); err != nil { + return errors.Wrap(err, "failed to update sidecar's status") + } + return nil + }); err != nil { logger.Error(err, "failed to update default sidecar cr") return admission.Errored(http.StatusInternalServerError, err) } @@ -176,6 +193,10 @@ func (r *Injector) ApplyToSidecar( if err := injector.InjectSidecar(unstructuredPodWithSidecar, unstructuredPod, sidecar, selector); err != nil { return errors.Wrap(err, "failed to inject sidecar") } + if err := runtime.DefaultUnstructuredConverter.FromUnstructured( + unstructuredPodWithSidecar.Object, podWithSidecar); err != nil { + return errors.Wrap(err, "failed to convert unstructured object to podWithSidecar") + } return nil } diff --git a/k8s/test/e2e/sidecar-demo/sidecar-with-default-sidecar.yaml b/k8s/test/e2e/sidecar-demo/sidecar-with-default-sidecar.yaml index eccbdd39..c1217ede 100644 --- a/k8s/test/e2e/sidecar-demo/sidecar-with-default-sidecar.yaml +++ b/k8s/test/e2e/sidecar-demo/sidecar-with-default-sidecar.yaml @@ -28,6 +28,7 @@ spec: annotations: sidecar.v6d.io/name: "default" labels: + app.kubernetes.io/instance: job-deployment-with-default-sidecar app: job-deployment-with-default-sidecar sidecar.v6d.io/enabled: "true" spec: diff --git a/k8s/test/e2e/sidecar/e2e.yaml b/k8s/test/e2e/sidecar/e2e.yaml index 21a1deca..f3d6ad69 100644 --- a/k8s/test/e2e/sidecar/e2e.yaml +++ b/k8s/test/e2e/sidecar/e2e.yaml @@ -21,7 +21,7 @@ setup: - name: download all sidecar images into kind cluster command: | make -C k8s/test/e2e publish-sidecar-images REGISTRY=localhost:5001 - - name: install app with default sidecar + - name: install app with default sidecar via vineyardctl command: | kubectl create namespace vineyard-job go run k8s/cmd/main.go inject -f k8s/test/e2e/sidecar-demo/sidecar-with-default-sidecar.yaml --apply-resources\ @@ -30,6 +30,16 @@ setup: - namespace: vineyard-job resource: deployment/job-deployment-with-default-sidecar for: condition=Available + - name: install app with default sidecar via kubectl + command: | + kubectl create namespace vineyard-job1 + kubectl label namespace vineyard-job1 sidecar-injection=enabled + sed -e 's/vineyard-job/vineyard-job1/g' k8s/test/e2e/sidecar-demo/sidecar-with-default-sidecar.yaml | \ + kubectl apply -f - + wait: + - namespace: vineyard-job1 + resource: deployment/job-deployment-with-default-sidecar + for: condition=Available - name: install app with custom sidecar command: | kubectl label namespace vineyard-job sidecar-injection=enabled @@ -60,6 +70,14 @@ verify: yq e '{"sum": .}' - | \ yq e 'to_entries' - expected: ../verify/values.yaml + - query: | + kubectl get pod -l app=job-deployment-with-default-sidecar -n vineyard-job1 -oname | \ + awk -F '/' '{print $2}' | \ + head -n 1 | \ + xargs kubectl logs -c job -n vineyard-job1 | \ + yq e '{"sum": .}' - | \ + yq e 'to_entries' - + expected: ../verify/values.yaml - query: | kubectl get pod -l app=job-deployment-with-custom-sidecar -n vineyard-job -oname | \ awk -F '/' '{print $2}' | \ diff --git a/k8s/test/hack/prepare-e2e.sh b/k8s/test/hack/prepare-e2e.sh index 2cea2f29..23306cf2 100644 --- a/k8s/test/hack/prepare-e2e.sh +++ b/k8s/test/hack/prepare-e2e.sh @@ -58,3 +58,11 @@ if ! command -v gomplate &> /dev/null; then fi fi +# v0.20.0 is not compatible on the github action +if [ $(kind version | awk '{print $2}') == "v0.20.0" ]; then + curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/v0.19.0/kind-linux-amd64 + chmod +x ./kind + mv ./kind /usr/local/bin/kind +fi + +