From dcf02193e487ce82e106f91f94a86fb1126998f9 Mon Sep 17 00:00:00 2001 From: Jacob Beacham Date: Fri, 5 May 2017 11:53:12 -0700 Subject: [PATCH] Convert kubeadm pull job to scenario. This also changes the --kubeadm boolean flag in kubernetes_e2e.py to an option to specify the type of job (ci, pull, periodic) so that it knows where to find the .deb artifacts created by the previous bazel job, since the output location can differ by type. --- jenkins/bootstrap_test.py | 13 ++++- jobs/config.json | 16 +++++- jobs/pull-kubernetes-e2e-kubeadm-gce.env | 15 +++++ jobs/pull-kubernetes-e2e-kubeadm-gce.sh | 73 ------------------------ prow/config.yaml | 4 ++ scenarios/kubernetes_e2e.py | 47 +++++++++------ scenarios/kubernetes_e2e_test.py | 52 +++++++++++++++-- 7 files changed, 120 insertions(+), 100 deletions(-) create mode 100755 jobs/pull-kubernetes-e2e-kubeadm-gce.env delete mode 100755 jobs/pull-kubernetes-e2e-kubeadm-gce.sh diff --git a/jenkins/bootstrap_test.py b/jenkins/bootstrap_test.py index 82f8cb749e14b..1c1b63fe65f73 100755 --- a/jenkins/bootstrap_test.py +++ b/jenkins/bootstrap_test.py @@ -1692,15 +1692,20 @@ def LoadProwYaml(self, path): if 'periodics' not in doc: self.fail('No periodics in prow config!') + if 'presubmits' not in doc: + self.fail('No presubmits in prow config!') + for item in doc.get('periodics'): self.AddProwJob(item) if 'postsubmits' not in doc: self.fail('No postsubmits in prow config!') + presubmits = doc.get('presubmits') postsubmits = doc.get('postsubmits') - for repo in postsubmits: - for job in postsubmits.get(repo): + + for repo, joblist in presubmits.items() + postsubmits.items(): + for job in joblist: self.AddProwJob(job) def LoadBootstrapYaml(self, path): @@ -2002,6 +2007,10 @@ def testValidJobEnvs(self): if 'gke' in job: stage = 'gs://kubernetes-release-dev/ci' suffix = True + elif 'kubeadm' in job: + # kubeadm-based jobs use out-of-band .deb artifacts, + # not the --stage flag. + continue else: stage = 'gs://kubernetes-release-pull/ci/%s' % job suffix = False diff --git a/jobs/config.json b/jobs/config.json index 4217210e7853c..feab14f63f310 100644 --- a/jobs/config.json +++ b/jobs/config.json @@ -2108,7 +2108,7 @@ "--cluster=", "--env-file=platforms/gce.env", "--env-file=jobs/ci-kubernetes-e2e-kubeadm-gce.env", - "--kubeadm", + "--kubeadm=ci", "--mode=local" ], "scenario": "kubernetes_e2e" @@ -2118,7 +2118,7 @@ "--cluster=", "--env-file=platforms/gce.env", "--env-file=jobs/ci-kubernetes-e2e-kubeadm-gce-1-6.env", - "--kubeadm", + "--kubeadm=ci", "--mode=local" ], "scenario": "kubernetes_e2e" @@ -2667,7 +2667,7 @@ "--cluster=", "--env-file=platforms/gce.env", "--env-file=jobs/periodic-kubernetes-e2e-kubeadm-gce-1-6.env", - "--kubeadm", + "--kubeadm=periodic", "--mode=local" ], "scenario": "kubernetes_e2e" @@ -2764,6 +2764,16 @@ ], "scenario": "kubernetes_e2e" }, + "pull-kubernetes-e2e-kubeadm-gce": { + "args": [ + "--cluster=", + "--env-file=platforms/gce.env", + "--env-file=jobs/pull-kubernetes-e2e-kubeadm-gce.env", + "--kubeadm=pull", + "--mode=local" + ], + "scenario": "kubernetes_e2e" + }, "pull-kubernetes-federation-e2e-gce": { "args": [ "--env-file=platforms/gce.env", diff --git a/jobs/pull-kubernetes-e2e-kubeadm-gce.env b/jobs/pull-kubernetes-e2e-kubeadm-gce.env new file mode 100755 index 0000000000000..e51802bbce817 --- /dev/null +++ b/jobs/pull-kubernetes-e2e-kubeadm-gce.env @@ -0,0 +1,15 @@ +### job-env + +PROJECT=k8s-jkns-pr-kubeadm +KUBERNETES_PROVIDER=kubernetes-anywhere + +GINKGO_PARALLEL=y +GINKGO_TEST_ARGS=--ginkgo.focus=\[Conformance\] --ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\] + +# Resource leak detection is disabled because prow runs multiple instances of +# this job in the same project concurrently, and resource leak detection will +# make the job flaky. +FAIL_ON_GCP_RESOURCE_LEAK=false + +# After post-env +KUBEKINS_TIMEOUT=55m diff --git a/jobs/pull-kubernetes-e2e-kubeadm-gce.sh b/jobs/pull-kubernetes-e2e-kubeadm-gce.sh deleted file mode 100755 index ce13ad66e3d81..0000000000000 --- a/jobs/pull-kubernetes-e2e-kubeadm-gce.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -# Copyright 2016 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail -set -o xtrace - -readonly testinfra="$(dirname "${0}")/.." - -### builder - -### job-env - -export PROJECT="k8s-jkns-pr-kubeadm" -export KUBERNETES_PROVIDER=kubernetes-anywhere -# These deliberately do not use the :- syntax so that the nounset flag produces -# succinct errors if either is unset. This job depends on the artifacts created -# during pull-kubernetes-bazel, so these env vars must be set correctly to find -# this pull's build artifacts. -export VERSION=${PULL_NUMBER}/${PULL_REFS} - -export E2E_NAME="e2e-kubeadm-${BUILD_NUMBER:-0}" -export E2E_OPT="--deployment kubernetes-anywhere --kubernetes-anywhere-path /workspace/kubernetes-anywhere" -export E2E_OPT+=" --kubernetes-anywhere-phase2-provider kubeadm --kubernetes-anywhere-cluster ${E2E_NAME}" -# The gs:// path given here should match jobs/pull-kubernetes-bazel.sh -export E2E_OPT+=" --kubernetes-anywhere-kubeadm-version gs://kubernetes-release-dev/bazel/${VERSION}/build/debs/" -export GINKGO_TEST_ARGS="--ginkgo.focus=\[Conformance\]" - -# Resource leak detection is disabled because prow runs multiple instances of -# this job in the same project concurrently, and resource leak detection will -# make the job flaky. -export FAIL_ON_GCP_RESOURCE_LEAK="false" - -### post-env - -# Assume we're upping, testing, and downing a cluster -export E2E_UP="${E2E_UP:-true}" -# TODO(pipejakob): Reenable testing when we have a pod network that works with -# kubeadm 1.6 clusters. For now, simply bringing up a cluster is a good e2e -# test, since it will only succeed if the apiserver is healthy and all -# expected nodes are registered. -export E2E_TEST="false" -export E2E_DOWN="${E2E_DOWN:-true}" - -# Skip gcloud update checking -export CLOUDSDK_COMPONENT_MANAGER_DISABLE_UPDATE_CHECK=true -# Use default component update behavior -export CLOUDSDK_EXPERIMENTAL_FAST_COMPONENT_UPDATE=false - -# Get golang into our PATH so we can run e2e.go -export PATH="${PATH}:/usr/local/go/bin" - -# After post-env -export GINKGO_PARALLEL="y" - -### Runner -readonly runner="/workspace/e2e-runner.sh" -export DOCKER_TIMEOUT="140m" -export KUBEKINS_TIMEOUT="120m" -"${runner}" diff --git a/prow/config.yaml b/prow/config.yaml index 91ec53a6599cd..56625d9ab4b61 100644 --- a/prow/config.yaml +++ b/prow/config.yaml @@ -91,6 +91,8 @@ presubmits: - "--pull=$(PULL_REFS)" - "--upload=gs://kubernetes-jenkins/pr-logs" - "--git-cache=/root/.cache/git" + - "--timeout=75" + - "--json" - "--clean" env: - name: GOOGLE_APPLICATION_CREDENTIALS @@ -252,6 +254,8 @@ presubmits: - "--pull=$(PULL_REFS)" - "--upload=gs://kubernetes-jenkins/pr-logs" - "--git-cache=/root/.cache/git" + - "--timeout=75" + - "--json" - "--clean" env: - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/scenarios/kubernetes_e2e.py b/scenarios/kubernetes_e2e.py index 670d56c7048b6..cd53a69b6899c 100755 --- a/scenarios/kubernetes_e2e.py +++ b/scenarios/kubernetes_e2e.py @@ -60,6 +60,32 @@ def kubekins(tag): return 'gcr.io/k8s-testimages/kubekins-e2e:%s' % tag +def kubeadm_version(mode): + """Return string to use for kubeadm version, given the job's mode (ci/pull/periodic).""" + version = '' + if mode in ['ci', 'periodic']: + # This job only runs against the kubernetes repo, and bootstrap.py leaves the + # current working directory at the repository root. Grab the SCM_REVISION so we + # can use the .debs built during the bazel-build job that should have already + # succeeded. + status = re.search( + r'STABLE_BUILD_SCM_REVISION ([^\n]+)', + check_output('hack/print-workspace-status.sh') + ) + if not status: + raise ValueError('STABLE_BUILD_SCM_REVISION not found') + version = status.group(1) + + elif mode == 'pull': + version = '%s/%s' % (os.environ['PULL_NUMBER'], os.getenv('PULL_REFS')) + + else: + raise ValueError("Unknown kubeadm mode given: %s" % mode) + + # The path given here should match jobs/ci-kubernetes-bazel-build.sh + return 'gs://kubernetes-release-dev/bazel/%s/bin/linux/amd64/' % version + + class LocalMode(object): """Runs e2e tests by calling e2e-runner.sh.""" def __init__(self, workspace): @@ -313,28 +339,15 @@ def main(args): cluster = args.cluster or 'e2e-gce-%s-%s' % ( os.environ['NODE_NAME'], os.getenv('EXECUTOR_NUMBER', 0)) - if args.kubeadm: + if args.kubeadm is not None: # Not from Jenkins cluster = args.cluster or 'e2e-kubeadm-%s' % os.getenv('BUILD_NUMBER', 0) - - # This job only runs against the kubernetes repo, and bootstrap.py leaves the - # current working directory at the repository root. Grab the SCM_REVISION so we - # can use the .debs built during the bazel-build job that should have already - # succeeded. - status = re.search( - r'STABLE_BUILD_SCM_REVISION ([^\n]+)', - check_output('hack/print-workspace-status.sh') - ) - if not status: - raise ValueError('STABLE_BUILD_SCM_REVISION not found') - + version = kubeadm_version(args.kubeadm) opt = '--deployment kubernetes-anywhere' \ ' --kubernetes-anywhere-path /workspace/kubernetes-anywhere' \ ' --kubernetes-anywhere-phase2-provider kubeadm' \ ' --kubernetes-anywhere-cluster %s' \ - ' --kubernetes-anywhere-kubeadm-version' \ - ' gs://kubernetes-release-dev/bazel/%s/build/debs/' % (cluster, status.group(1)) - # The gs:// path given here should match jobs/ci-kubernetes-bazel-build.sh + ' --kubernetes-anywhere-kubeadm-version %s' % (cluster, version) mode.add_environment('E2E_OPT=%s' % opt) # env blacklist. @@ -435,7 +448,7 @@ def create_parser(): parser.add_argument( '--down', default='true', help='If we need to set --down in e2e.go') parser.add_argument( - '--kubeadm', action='store_true', help='If the test is a kubeadm job') + '--kubeadm', choices=['ci', 'periodic', 'pull']) parser.add_argument( '--soak-test', action='store_true', help='If the test is a soak test job') parser.add_argument( diff --git a/scenarios/kubernetes_e2e_test.py b/scenarios/kubernetes_e2e_test.py index efe43a0d45345..f8a88b0b1facb 100755 --- a/scenarios/kubernetes_e2e_test.py +++ b/scenarios/kubernetes_e2e_test.py @@ -20,6 +20,7 @@ """Test for kubernetes_e2e.py""" import json +import os import re import shutil import string @@ -111,17 +112,18 @@ def test_local(self): for call in self.callstack: self.assertFalse(call.startswith('docker')) - def test_kubeadm(self): - """Make sure kubeadm mode is fine overall.""" - args = self.parser.parse_args(['--mode=local', '--kubeadm']) + def test_kubeadm_ci(self): + """Make sure kubeadm ci mode is fine overall.""" + args = self.parser.parse_args(['--mode=local', '--kubeadm=ci']) self.assertEqual(args.mode, 'local') - self.assertEqual(args.kubeadm, True) + self.assertEqual(args.kubeadm, 'ci') with Stub(kubernetes_e2e, 'check_env', self.fake_check_env): with Stub(kubernetes_e2e, 'check_output', self.fake_output_work_status): kubernetes_e2e.main(args) self.assertIn('E2E_OPT', self.envs) - self.assertIn('v1.7.0-alpha.0.1320+599539dc0b9997', self.envs['E2E_OPT']) + self.assertIn('--kubernetes-anywhere-kubeadm-version gs://kubernetes-release-dev/bazel/' + 'v1.7.0-alpha.0.1320+599539dc0b9997/bin/linux/amd64/', self.envs['E2E_OPT']) called = False for call in self.callstack: self.assertFalse(call.startswith('docker')) @@ -129,6 +131,46 @@ def test_kubeadm(self): called = True self.assertTrue(called) + def test_kubeadm_periodic(self): + """Make sure kubeadm periodic mode is fine overall.""" + args = self.parser.parse_args(['--mode=local', '--kubeadm=periodic']) + self.assertEqual(args.mode, 'local') + self.assertEqual(args.kubeadm, 'periodic') + with Stub(kubernetes_e2e, 'check_env', self.fake_check_env): + with Stub(kubernetes_e2e, 'check_output', self.fake_output_work_status): + kubernetes_e2e.main(args) + + self.assertIn('E2E_OPT', self.envs) + self.assertIn('--kubernetes-anywhere-kubeadm-version gs://kubernetes-release-dev/bazel/' + 'v1.7.0-alpha.0.1320+599539dc0b9997/bin/linux/amd64/', self.envs['E2E_OPT']) + called = False + for call in self.callstack: + self.assertFalse(call.startswith('docker')) + if call == 'hack/print-workspace-status.sh': + called = True + self.assertTrue(called) + + def test_kubeadm_pull(self): + """Make sure kubeadm pull mode is fine overall.""" + args = self.parser.parse_args(['--mode=local', '--kubeadm=pull']) + self.assertEqual(args.mode, 'local') + self.assertEqual(args.kubeadm, 'pull') + fake_env = {'PULL_NUMBER': 1234, 'PULL_REFS': 'master:abcd'} + with Stub(kubernetes_e2e, 'check_env', self.fake_check_env): + with Stub(os, 'environ', fake_env): + kubernetes_e2e.main(args) + + self.assertIn('E2E_OPT', self.envs) + self.assertIn('--kubernetes-anywhere-kubeadm-version gs://kubernetes-release-dev/bazel/' + '1234/master:abcd/bin/linux/amd64/', self.envs['E2E_OPT']) + + def test_kubeadm_invalid(self): + """Make sure kubeadm invalid mode exits unsuccessfully.""" + with self.assertRaises(SystemExit) as sysexit: + self.parser.parse_args(['--mode=local', '--kubeadm=deploy']) + + self.assertEqual(sysexit.exception.code, 2) + class DockerTest(ScenarioTest): """Class for testing e2e scenario in docker mode.""" def test_docker(self):