From 16aa06cb46ed13875bba5bf0b797115ce0624610 Mon Sep 17 00:00:00 2001 From: Dane LeBlanc Date: Fri, 23 Feb 2018 15:08:22 -0500 Subject: [PATCH] ci-kubernetes-e2e-kubeadm-gce pulling ci/latest not bazel build artifacts The ci-kubernetes-e2e-kubeadm-gce test jobs are consistently failing. If you look at test results and logs for a given failing test job, and then compare it to the corresponding prior (prerequisite) bazel build job, you can see that the build job is pushing the build artifacts in the proper gs://kubernetes-release/bazel/... storage bucket, but the test job is extracting (or attempting to extract) build results from ci/latest. The test job also seems to be using inconsistent versions of kubeadm/kubelet/kubernetes. This is very likely one of the causes of the CI test outages described in https://github.com/kubernetes/kubernetes/issues/59762. This failure mode is also seen in other ci-kubernetes-e2e-XXXX test cases, but I'd like to try a fix first on one representative test job, and then replicate the fix to other test jobs if it works. Fixes #6978 --- jobs/config.json | 3 --- scenarios/kubernetes_e2e.py | 7 ++++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/jobs/config.json b/jobs/config.json index 4e8209e24288a..181d9cfef8afa 100644 --- a/jobs/config.json +++ b/jobs/config.json @@ -8771,12 +8771,9 @@ "--cluster=", "--deployment=kubernetes-anywhere", "--env-file=jobs/platform/gce.env", - "--extract=ci/latest", "--gcp-zone=us-central1-f", "--kubeadm=ci", "--kubernetes-anywhere-dump-cluster-logs=true", - "--kubernetes-anywhere-kubelet-ci-version=latest", - "--kubernetes-anywhere-kubernetes-version=ci/latest", "--provider=kubernetes-anywhere", "--test_args=--ginkgo.focus=\\[Conformance\\]|\\[Feature:BootstrapTokens\\]|\\[Feature:NodeAuthorizer\\] --minStartupPods=8", "--timeout=300m" diff --git a/scenarios/kubernetes_e2e.py b/scenarios/kubernetes_e2e.py index 4e219e7527ca3..71be4e02d919f 100755 --- a/scenarios/kubernetes_e2e.py +++ b/scenarios/kubernetes_e2e.py @@ -547,9 +547,10 @@ def main(args): '--kubernetes-anywhere-kubeadm-version=%s' % version, ]) - if args.kubeadm == "pull": - # If this is a pull job; the kubelet version should equal - # the kubeadm version here: we should use debs from the PR build + if args.kubeadm == "pull" or args.kubeadm == "ci": + # If this is a pull or ci job; the kubelet version should equal + # the kubeadm version here: we should use debs from the PR or + # post-submit build runner_args.extend([ '--kubernetes-anywhere-kubelet-version=%s' % version, ])