diff --git a/bootstrap/kubeadm/config/manager/manager.yaml b/bootstrap/kubeadm/config/manager/manager.yaml index 233ba3d5fd90..a01ea2e1308f 100644 --- a/bootstrap/kubeadm/config/manager/manager.yaml +++ b/bootstrap/kubeadm/config/manager/manager.yaml @@ -33,6 +33,8 @@ spec: path: /readyz port: healthz livenessProbe: + failureThreshold: 12 + periodSeconds: 10 httpGet: path: /healthz port: healthz diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 6e4d23e7ad32..e3b2a9c166b0 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -34,6 +34,8 @@ spec: path: /readyz port: healthz livenessProbe: + failureThreshold: 12 + periodSeconds: 10 httpGet: path: /healthz port: healthz diff --git a/controlplane/kubeadm/config/manager/manager.yaml b/controlplane/kubeadm/config/manager/manager.yaml index b5e31734e031..2363bd55d3ba 100644 --- a/controlplane/kubeadm/config/manager/manager.yaml +++ b/controlplane/kubeadm/config/manager/manager.yaml @@ -33,6 +33,8 @@ spec: path: /readyz port: healthz livenessProbe: + failureThreshold: 12 + periodSeconds: 10 httpGet: path: /healthz port: healthz diff --git a/go.mod b/go.mod index 8709e0227229..0d887b266f17 100644 --- a/go.mod +++ b/go.mod @@ -41,3 +41,5 @@ require ( sigs.k8s.io/controller-runtime v0.10.3-0.20211011182302-43ea648ec318 sigs.k8s.io/yaml v1.3.0 ) + +replace sigs.k8s.io/controller-runtime => github.com/fabriziopandini/controller-runtime v0.2.0-beta.1.0.20211105090959-708d5397c4f3 diff --git a/go.sum b/go.sum index f795c572bce9..0a1e92008423 100644 --- a/go.sum +++ b/go.sum @@ -165,6 +165,8 @@ github.com/evanphx/json-patch v4.11.0+incompatible h1:glyUF9yIYtMHzn8xaKw5rMhdWc github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= +github.com/fabriziopandini/controller-runtime v0.2.0-beta.1.0.20211105090959-708d5397c4f3 h1:MMRZoNdbvMkVb5d+opdF6nqAmQVPsOA+GWd3Q2fQRo0= +github.com/fabriziopandini/controller-runtime v0.2.0-beta.1.0.20211105090959-708d5397c4f3/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= @@ -1134,8 +1136,6 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/controller-runtime v0.10.3-0.20211011182302-43ea648ec318 h1:MxgxYdJH4JJcEkyzFwYeH2o8WAlRlWE4cuIfsMvqK5g= -sigs.k8s.io/controller-runtime v0.10.3-0.20211011182302-43ea648ec318/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY= sigs.k8s.io/kustomize/api v0.8.11 h1:LzQzlq6Z023b+mBtc6v72N2mSHYmN8x7ssgbf/hv0H8= sigs.k8s.io/kustomize/api v0.8.11/go.mod h1:a77Ls36JdfCWojpUqR6m60pdGY1AYFix4AH83nJtY1g= sigs.k8s.io/kustomize/cmd/config v0.9.13/go.mod h1:7547FLF8W/lTaDf0BDqFTbZxM9zqwEJqCKN9sSR0xSs= diff --git a/test/e2e/config/docker.yaml b/test/e2e/config/docker.yaml index fcb7fc2d8e65..20e902ef8bef 100644 --- a/test/e2e/config/docker.yaml +++ b/test/e2e/config/docker.yaml @@ -34,6 +34,8 @@ providers: type: "url" contract: v1alpha3 replacements: + - old: "--leader-elect" + new: "--leader-elect=false" - old: --metrics-addr=127.0.0.1:8080 new: --metrics-addr=:8080 files: @@ -43,6 +45,8 @@ providers: type: "url" contract: v1alpha4 replacements: + - old: "--leader-elect" + new: "--leader-elect=false" - old: --metrics-addr=127.0.0.1:8080 new: --metrics-addr=:8080 files: @@ -50,6 +54,8 @@ providers: - name: v1.0.99 # next; use manifest from source files value: ../../../config/default replacements: + - old: "--leader-elect" + new: "--leader-elect=false" - old: --metrics-bind-addr=localhost:8080 new: --metrics-bind-addr=:8080 files: @@ -63,6 +69,8 @@ providers: type: "url" contract: v1alpha3 replacements: + - old: "--leader-elect" + new: "--leader-elect=false" - old: --metrics-addr=127.0.0.1:8080 new: --metrics-addr=:8080 files: @@ -72,6 +80,8 @@ providers: type: "url" contract: v1alpha4 replacements: + - old: "--leader-elect" + new: "--leader-elect=false" - old: --metrics-addr=127.0.0.1:8080 new: --metrics-addr=:8080 files: @@ -79,6 +89,8 @@ providers: - name: v1.0.99 # next; use manifest from source files value: ../../../bootstrap/kubeadm/config/default replacements: + - old: "--leader-elect" + new: "--leader-elect=false" - old: --metrics-bind-addr=localhost:8080 new: --metrics-bind-addr=:8080 files: @@ -92,6 +104,8 @@ providers: type: "url" contract: v1alpha3 replacements: + - old: "--leader-elect" + new: "--leader-elect=false" - old: --metrics-addr=127.0.0.1:8080 new: --metrics-addr=:8080 files: @@ -101,6 +115,8 @@ providers: type: "url" contract: v1alpha4 replacements: + - old: "--leader-elect" + new: "--leader-elect=false" - old: --metrics-addr=127.0.0.1:8080 new: --metrics-addr=:8080 files: @@ -108,6 +124,8 @@ providers: - name: v1.0.99 # next; use manifest from source files value: ../../../controlplane/kubeadm/config/default replacements: + - old: "--leader-elect" + new: "--leader-elect=false" - old: --metrics-bind-addr=localhost:8080 new: --metrics-bind-addr=:8080 files: @@ -121,6 +139,8 @@ providers: type: "url" contract: v1alpha3 replacements: + - old: "--leader-elect" + new: "--leader-elect=false" - old: --metrics-addr=127.0.0.1:8080 new: --metrics-addr=:8080 files: @@ -131,6 +151,8 @@ providers: type: "url" contract: v1alpha4 replacements: + - old: "--leader-elect" + new: "--leader-elect=false" - old: --metrics-addr=127.0.0.1:8080 new: --metrics-addr=:8080 files: @@ -139,6 +161,8 @@ providers: - name: v1.0.99 # next; use manifest from source files value: ../../../test/infrastructure/docker/config/default replacements: + - old: "--leader-elect" + new: "--leader-elect=false" - old: --metrics-bind-addr=localhost:8080 new: --metrics-bind-addr=:8080 files: @@ -180,9 +204,9 @@ variables: CLUSTER_TOPOLOGY: "true" # NOTE: INIT_WITH_BINARY and INIT_WITH_KUBERNETES_VERSION are only used by the clusterctl upgrade test to initialize # the management cluster to be upgraded. - INIT_WITH_BINARY: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.4/clusterctl-{OS}-{ARCH}" - INIT_WITH_PROVIDERS_CONTRACT: "v1alpha4" - INIT_WITH_KUBERNETES_VERSION: "v1.22.0" + INIT_WITH_BINARY: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.23/clusterctl-{OS}-{ARCH}" + INIT_WITH_PROVIDERS_CONTRACT: "v1alpha3" + INIT_WITH_KUBERNETES_VERSION: "v1.21.2" intervals: default/wait-controllers: ["3m", "10s"] diff --git a/test/framework/alltypes_helpers.go b/test/framework/alltypes_helpers.go index 4f38d18f6216..30b7a6ffce83 100644 --- a/test/framework/alltypes_helpers.go +++ b/test/framework/alltypes_helpers.go @@ -26,7 +26,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" @@ -72,6 +71,36 @@ func GetCAPIResources(ctx context.Context, input GetCAPIResourcesInput) []*unstr } } + // cluster-wide + types = []metav1.TypeMeta{ + { + Kind: "Deployment", + APIVersion: "apps/v1", + }, + { + Kind: "Pod", + APIVersion: "v1", + }, + } + + for i := range types { + typeMeta := types[i] + typeList := new(unstructured.UnstructuredList) + typeList.SetAPIVersion(typeMeta.APIVersion) + typeList.SetKind(typeMeta.Kind) + + if err := input.Lister.List(ctx, typeList); err != nil { + if apierrors.IsNotFound(err) { + continue + } + Fail(fmt.Sprintf("failed to list %q resources: %v", typeList.GroupVersionKind(), err)) + } + for i := range typeList.Items { + obj := typeList.Items[i] + objList = append(objList, &obj) + } + } + return objList } diff --git a/test/framework/clusterctl/clusterctl_helpers.go b/test/framework/clusterctl/clusterctl_helpers.go index f016172ec7c0..5ed6618c6957 100644 --- a/test/framework/clusterctl/clusterctl_helpers.go +++ b/test/framework/clusterctl/clusterctl_helpers.go @@ -99,6 +99,9 @@ func InitManagementClusterAndWatchControllerLogs(ctx context.Context, input Init framework.WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{ Getter: client, Deployment: deployment, + GetLister: client, + ClientSet: input.ClusterProxy.GetClientSet(), + LogPath: filepath.Join(input.LogFolder, "controllers"), }, intervals...) // Start streaming logs from all controller providers @@ -155,6 +158,9 @@ func UpgradeManagementClusterAndWait(ctx context.Context, input UpgradeManagemen framework.WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{ Getter: client, Deployment: deployment, + GetLister: client, + ClientSet: input.ClusterProxy.GetClientSet(), + LogPath: filepath.Join(input.LogFolder, "controllers"), }, intervals...) // Start streaming logs from all controller providers diff --git a/test/framework/deployment_helpers.go b/test/framework/deployment_helpers.go index 2652e56d10c9..f7c5b19befdc 100644 --- a/test/framework/deployment_helpers.go +++ b/test/framework/deployment_helpers.go @@ -47,6 +47,9 @@ import ( type WaitForDeploymentsAvailableInput struct { Getter Getter Deployment *appsv1.Deployment + GetLister GetLister + ClientSet *kubernetes.Clientset + LogPath string } // WaitForDeploymentsAvailable waits until the Deployment has status.Available = True, that signals that @@ -69,7 +72,17 @@ func WaitForDeploymentsAvailable(ctx context.Context, input WaitForDeploymentsAv } } return false - }, intervals...).Should(BeTrue(), func() string { return DescribeFailedDeployment(input, deployment) }) + }, intervals...).Should(BeTrue(), func() string { + if input.GetLister != nil && input.ClientSet != nil && input.LogPath != "" { + GetDeploymentLogs(ctx, GetDeploymentLogsInput{ + Deployment: deployment, + GetLister: input.GetLister, + ClientSet: input.ClientSet, + LogPath: input.LogPath, + }) + } + return DescribeFailedDeployment(input, deployment) + }) } // DescribeFailedDeployment returns detailed output to help debug a deployment failure in e2e. @@ -152,6 +165,68 @@ func WatchDeploymentLogs(ctx context.Context, input WatchDeploymentLogsInput) { } } +type GetDeploymentLogsInput struct { + GetLister GetLister + ClientSet *kubernetes.Clientset + Deployment *appsv1.Deployment + LogPath string +} + +func GetDeploymentLogs(ctx context.Context, input GetDeploymentLogsInput) { + Expect(ctx).NotTo(BeNil(), "ctx is required for GetDeploymentLogs") + Expect(input.ClientSet).NotTo(BeNil(), "input.ClientSet is required for GetDeploymentLogs") + Expect(input.Deployment).NotTo(BeNil(), "input.Deployment is required for GetDeploymentLogs") + + deployment := &appsv1.Deployment{} + key := client.ObjectKeyFromObject(input.Deployment) + Expect(input.GetLister.Get(ctx, key, deployment)).To(Succeed(), "Failed to get deployment %s/%s", input.Deployment.Namespace, input.Deployment.Name) + + selector, err := metav1.LabelSelectorAsMap(deployment.Spec.Selector) + Expect(err).NotTo(HaveOccurred(), "Failed to Pods selector for deployment %s/%s", input.Deployment.Namespace, input.Deployment.Name) + + pods := &corev1.PodList{} + Expect(input.GetLister.List(ctx, pods, client.InNamespace(input.Deployment.Namespace), client.MatchingLabels(selector))).To(Succeed(), "Failed to list Pods for deployment %s/%s", input.Deployment.Namespace, input.Deployment.Name) + + for _, pod := range pods.Items { + for _, container := range deployment.Spec.Template.Spec.Containers { + log.Logf("Creating log watcher for controller %s/%s, pod %s, container %s", input.Deployment.Namespace, input.Deployment.Name, pod.Name, container.Name) + + // Watch each container's logs in a goroutine so we can stream them all concurrently. + go func(pod corev1.Pod, container corev1.Container) { + defer GinkgoRecover() + + logFile := filepath.Clean(path.Join(input.LogPath, input.Deployment.Name, pod.Name, container.Name+".log")) + Expect(os.MkdirAll(filepath.Dir(logFile), 0750)).To(Succeed()) + + f, err := os.OpenFile(logFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) + Expect(err).NotTo(HaveOccurred()) + defer f.Close() + + opts := &corev1.PodLogOptions{ + Container: container.Name, + Follow: false, + } + + podLogs, err := input.ClientSet.CoreV1().Pods(input.Deployment.Namespace).GetLogs(pod.Name, opts).Stream(ctx) + if err != nil { + // Failing to stream logs should not cause the test to fail + log.Logf("Error starting logs stream for pod %s/%s, container %s: %v", input.Deployment.Namespace, pod.Name, container.Name, err) + return + } + defer podLogs.Close() + + out := bufio.NewWriter(f) + defer out.Flush() + _, err = out.ReadFrom(podLogs) + if err != nil && err != io.ErrUnexpectedEOF { + // Failing to stream logs should not cause the test to fail + log.Logf("Got error while streaming logs for pod %s/%s, container %s: %v", input.Deployment.Namespace, pod.Name, container.Name, err) + } + }(pod, container) + } + } +} + type WatchPodMetricsInput struct { GetLister GetLister ClientSet *kubernetes.Clientset diff --git a/test/go.mod b/test/go.mod index 0f0f587ed90f..d5f2c482a4fc 100644 --- a/test/go.mod +++ b/test/go.mod @@ -28,3 +28,5 @@ require ( sigs.k8s.io/kind v0.11.1 sigs.k8s.io/yaml v1.3.0 ) + +replace sigs.k8s.io/controller-runtime => github.com/fabriziopandini/controller-runtime v0.2.0-beta.1.0.20211105090959-708d5397c4f3 diff --git a/test/go.sum b/test/go.sum index 67f7a7222a7d..0245ffd67de3 100644 --- a/test/go.sum +++ b/test/go.sum @@ -315,6 +315,8 @@ github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQL github.com/evanphx/json-patch/v5 v5.2.0 h1:8ozOH5xxoMYDt5/u+yMTsVXydVCbTORFnOOoq2lumco= github.com/evanphx/json-patch/v5 v5.2.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= +github.com/fabriziopandini/controller-runtime v0.2.0-beta.1.0.20211105090959-708d5397c4f3 h1:MMRZoNdbvMkVb5d+opdF6nqAmQVPsOA+GWd3Q2fQRo0= +github.com/fabriziopandini/controller-runtime v0.2.0-beta.1.0.20211105090959-708d5397c4f3/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= @@ -1477,8 +1479,6 @@ rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/controller-runtime v0.10.3-0.20211011182302-43ea648ec318 h1:MxgxYdJH4JJcEkyzFwYeH2o8WAlRlWE4cuIfsMvqK5g= -sigs.k8s.io/controller-runtime v0.10.3-0.20211011182302-43ea648ec318/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY= sigs.k8s.io/kind v0.11.1 h1:pVzOkhUwMBrCB0Q/WllQDO3v14Y+o2V0tFgjTqIUjwA= sigs.k8s.io/kind v0.11.1/go.mod h1:fRpgVhtqAWrtLB9ED7zQahUimpUXuG/iHT88xYqEGIA= sigs.k8s.io/kustomize/api v0.8.11/go.mod h1:a77Ls36JdfCWojpUqR6m60pdGY1AYFix4AH83nJtY1g= diff --git a/test/infrastructure/docker/config/manager/manager.yaml b/test/infrastructure/docker/config/manager/manager.yaml index 4454c900483b..b27cfa51aacc 100644 --- a/test/infrastructure/docker/config/manager/manager.yaml +++ b/test/infrastructure/docker/config/manager/manager.yaml @@ -31,6 +31,8 @@ spec: path: /readyz port: healthz livenessProbe: + failureThreshold: 12 + periodSeconds: 10 httpGet: path: /healthz port: healthz