From 5cda592b2c8ba9bb2f9900c4dbda6fb763fb2068 Mon Sep 17 00:00:00 2001 From: Jacob Beacham Date: Thu, 13 Apr 2017 13:54:10 -0700 Subject: [PATCH 1/2] Use a consistent build id between bazel/kubeadm pull jobs. Since the kubeadm e2e job uses the CI artifacts pushed by the bazel job, make sure we use a consistent id for their storage (the Kubernetes version isn't stable for pull jobs since each job pulls and applies the PR at different times, resulting in different commit SHAs). --- jobs/pull-kubernetes-bazel.sh | 6 +++--- jobs/pull-kubernetes-e2e-kubeadm-gce.sh | 13 ++++++------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/jobs/pull-kubernetes-bazel.sh b/jobs/pull-kubernetes-bazel.sh index 137a7ae3b195..8baf8c95703e 100755 --- a/jobs/pull-kubernetes-bazel.sh +++ b/jobs/pull-kubernetes-bazel.sh @@ -34,11 +34,11 @@ if [[ "${rc}" == 0 ]]; then fi if [[ "${rc}" == 0 ]]; then - version=$(cat bazel-genfiles/version || true) - if [[ -z "${version}" ]]; then - echo "Kubernetes version missing; not uploading ci artifacts." + if [[ -z "${PULL_NUMBER:-}" || -z "${PULL_REFS:-}" ]]; then + echo "\$PULL_NUMBER or \$PULL_REFS is empty; not uploading ci artifacts." rc=1 else + version=${PULL_NUMBER:-}/${PULL_REFS:-} bazel run //:ci-artifacts -- "gs://kubernetes-release-dev/bazel/${version}" && rc=$? || rc=$? fi fi diff --git a/jobs/pull-kubernetes-e2e-kubeadm-gce.sh b/jobs/pull-kubernetes-e2e-kubeadm-gce.sh index d8144308d638..ce13ad66e3d8 100755 --- a/jobs/pull-kubernetes-e2e-kubeadm-gce.sh +++ b/jobs/pull-kubernetes-e2e-kubeadm-gce.sh @@ -26,18 +26,17 @@ readonly testinfra="$(dirname "${0}")/.." export PROJECT="k8s-jkns-pr-kubeadm" export KUBERNETES_PROVIDER=kubernetes-anywhere - -# This job only runs against the kubernetes repo, and bootstrap.py leaves the -# current working directory at the repository root. Grab the SCM_REVISION so we -# can use the .debs built during the bazel-build job that should have already -# succeeded. -export SCM_VERSION=$(./hack/print-workspace-status.sh | grep ^STABLE_BUILD_SCM_REVISION | cut -d' ' -f2) +# These deliberately do not use the :- syntax so that the nounset flag produces +# succinct errors if either is unset. This job depends on the artifacts created +# during pull-kubernetes-bazel, so these env vars must be set correctly to find +# this pull's build artifacts. +export VERSION=${PULL_NUMBER}/${PULL_REFS} export E2E_NAME="e2e-kubeadm-${BUILD_NUMBER:-0}" export E2E_OPT="--deployment kubernetes-anywhere --kubernetes-anywhere-path /workspace/kubernetes-anywhere" export E2E_OPT+=" --kubernetes-anywhere-phase2-provider kubeadm --kubernetes-anywhere-cluster ${E2E_NAME}" # The gs:// path given here should match jobs/pull-kubernetes-bazel.sh -export E2E_OPT+=" --kubernetes-anywhere-kubeadm-version gs://kubernetes-release-dev/bazel/${SCM_VERSION}/build/debs/" +export E2E_OPT+=" --kubernetes-anywhere-kubeadm-version gs://kubernetes-release-dev/bazel/${VERSION}/build/debs/" export GINKGO_TEST_ARGS="--ginkgo.focus=\[Conformance\]" # Resource leak detection is disabled because prow runs multiple instances of From c0287dd352f9e25ca99088ace2d8ecf2f462dba2 Mon Sep 17 00:00:00 2001 From: Jacob Beacham Date: Fri, 5 May 2017 11:53:12 -0700 Subject: [PATCH 2/2] Convert kubeadm pull job to scenario. This also changes the --kubeadm boolean flag in kubernetes_e2e.py to an option to specify the type of job (ci, pull, periodic) so that it knows where to find the .deb artifacts created by the previous bazel job, since the output location can differ by type. --- jenkins/bootstrap_test.py | 13 ++++- jobs/config.json | 16 +++++- jobs/pull-kubernetes-e2e-kubeadm-gce.env | 15 +++++ jobs/pull-kubernetes-e2e-kubeadm-gce.sh | 73 ------------------------ prow/config.yaml | 4 ++ scenarios/kubernetes_e2e.py | 45 +++++++++------ scenarios/kubernetes_e2e_test.py | 53 +++++++++++++++-- 7 files changed, 120 insertions(+), 99 deletions(-) create mode 100755 jobs/pull-kubernetes-e2e-kubeadm-gce.env delete mode 100755 jobs/pull-kubernetes-e2e-kubeadm-gce.sh diff --git a/jenkins/bootstrap_test.py b/jenkins/bootstrap_test.py index 82f8cb749e14..1c1b63fe65f7 100755 --- a/jenkins/bootstrap_test.py +++ b/jenkins/bootstrap_test.py @@ -1692,15 +1692,20 @@ def LoadProwYaml(self, path): if 'periodics' not in doc: self.fail('No periodics in prow config!') + if 'presubmits' not in doc: + self.fail('No presubmits in prow config!') + for item in doc.get('periodics'): self.AddProwJob(item) if 'postsubmits' not in doc: self.fail('No postsubmits in prow config!') + presubmits = doc.get('presubmits') postsubmits = doc.get('postsubmits') - for repo in postsubmits: - for job in postsubmits.get(repo): + + for repo, joblist in presubmits.items() + postsubmits.items(): + for job in joblist: self.AddProwJob(job) def LoadBootstrapYaml(self, path): @@ -2002,6 +2007,10 @@ def testValidJobEnvs(self): if 'gke' in job: stage = 'gs://kubernetes-release-dev/ci' suffix = True + elif 'kubeadm' in job: + # kubeadm-based jobs use out-of-band .deb artifacts, + # not the --stage flag. + continue else: stage = 'gs://kubernetes-release-pull/ci/%s' % job suffix = False diff --git a/jobs/config.json b/jobs/config.json index bc5f5610b630..28e72a12542e 100644 --- a/jobs/config.json +++ b/jobs/config.json @@ -2112,7 +2112,7 @@ "--cluster=", "--env-file=platforms/gce.env", "--env-file=jobs/ci-kubernetes-e2e-kubeadm-gce.env", - "--kubeadm", + "--kubeadm=ci", "--mode=local" ], "scenario": "kubernetes_e2e" @@ -2122,7 +2122,7 @@ "--cluster=", "--env-file=platforms/gce.env", "--env-file=jobs/ci-kubernetes-e2e-kubeadm-gce-1-6.env", - "--kubeadm", + "--kubeadm=ci", "--mode=local" ], "scenario": "kubernetes_e2e" @@ -2705,7 +2705,7 @@ "--cluster=", "--env-file=platforms/gce.env", "--env-file=jobs/periodic-kubernetes-e2e-kubeadm-gce-1-6.env", - "--kubeadm", + "--kubeadm=periodic", "--mode=local" ], "scenario": "kubernetes_e2e" @@ -2802,6 +2802,16 @@ ], "scenario": "kubernetes_e2e" }, + "pull-kubernetes-e2e-kubeadm-gce": { + "args": [ + "--cluster=", + "--env-file=platforms/gce.env", + "--env-file=jobs/pull-kubernetes-e2e-kubeadm-gce.env", + "--kubeadm=pull", + "--mode=local" + ], + "scenario": "kubernetes_e2e" + }, "pull-kubernetes-federation-e2e-gce": { "args": [ "--env-file=platforms/gce.env", diff --git a/jobs/pull-kubernetes-e2e-kubeadm-gce.env b/jobs/pull-kubernetes-e2e-kubeadm-gce.env new file mode 100755 index 000000000000..e51802bbce81 --- /dev/null +++ b/jobs/pull-kubernetes-e2e-kubeadm-gce.env @@ -0,0 +1,15 @@ +### job-env + +PROJECT=k8s-jkns-pr-kubeadm +KUBERNETES_PROVIDER=kubernetes-anywhere + +GINKGO_PARALLEL=y +GINKGO_TEST_ARGS=--ginkgo.focus=\[Conformance\] --ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\] + +# Resource leak detection is disabled because prow runs multiple instances of +# this job in the same project concurrently, and resource leak detection will +# make the job flaky. +FAIL_ON_GCP_RESOURCE_LEAK=false + +# After post-env +KUBEKINS_TIMEOUT=55m diff --git a/jobs/pull-kubernetes-e2e-kubeadm-gce.sh b/jobs/pull-kubernetes-e2e-kubeadm-gce.sh deleted file mode 100755 index ce13ad66e3d8..000000000000 --- a/jobs/pull-kubernetes-e2e-kubeadm-gce.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -# Copyright 2016 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail -set -o xtrace - -readonly testinfra="$(dirname "${0}")/.." - -### builder - -### job-env - -export PROJECT="k8s-jkns-pr-kubeadm" -export KUBERNETES_PROVIDER=kubernetes-anywhere -# These deliberately do not use the :- syntax so that the nounset flag produces -# succinct errors if either is unset. This job depends on the artifacts created -# during pull-kubernetes-bazel, so these env vars must be set correctly to find -# this pull's build artifacts. -export VERSION=${PULL_NUMBER}/${PULL_REFS} - -export E2E_NAME="e2e-kubeadm-${BUILD_NUMBER:-0}" -export E2E_OPT="--deployment kubernetes-anywhere --kubernetes-anywhere-path /workspace/kubernetes-anywhere" -export E2E_OPT+=" --kubernetes-anywhere-phase2-provider kubeadm --kubernetes-anywhere-cluster ${E2E_NAME}" -# The gs:// path given here should match jobs/pull-kubernetes-bazel.sh -export E2E_OPT+=" --kubernetes-anywhere-kubeadm-version gs://kubernetes-release-dev/bazel/${VERSION}/build/debs/" -export GINKGO_TEST_ARGS="--ginkgo.focus=\[Conformance\]" - -# Resource leak detection is disabled because prow runs multiple instances of -# this job in the same project concurrently, and resource leak detection will -# make the job flaky. -export FAIL_ON_GCP_RESOURCE_LEAK="false" - -### post-env - -# Assume we're upping, testing, and downing a cluster -export E2E_UP="${E2E_UP:-true}" -# TODO(pipejakob): Reenable testing when we have a pod network that works with -# kubeadm 1.6 clusters. For now, simply bringing up a cluster is a good e2e -# test, since it will only succeed if the apiserver is healthy and all -# expected nodes are registered. -export E2E_TEST="false" -export E2E_DOWN="${E2E_DOWN:-true}" - -# Skip gcloud update checking -export CLOUDSDK_COMPONENT_MANAGER_DISABLE_UPDATE_CHECK=true -# Use default component update behavior -export CLOUDSDK_EXPERIMENTAL_FAST_COMPONENT_UPDATE=false - -# Get golang into our PATH so we can run e2e.go -export PATH="${PATH}:/usr/local/go/bin" - -# After post-env -export GINKGO_PARALLEL="y" - -### Runner -readonly runner="/workspace/e2e-runner.sh" -export DOCKER_TIMEOUT="140m" -export KUBEKINS_TIMEOUT="120m" -"${runner}" diff --git a/prow/config.yaml b/prow/config.yaml index 88eaf591174d..788951e13687 100644 --- a/prow/config.yaml +++ b/prow/config.yaml @@ -91,6 +91,8 @@ presubmits: - "--pull=$(PULL_REFS)" - "--upload=gs://kubernetes-jenkins/pr-logs" - "--git-cache=/root/.cache/git" + - "--timeout=75" + - "--json" - "--clean" env: - name: GOOGLE_APPLICATION_CREDENTIALS @@ -251,6 +253,8 @@ presubmits: - "--pull=$(PULL_REFS)" - "--upload=gs://kubernetes-jenkins/pr-logs" - "--git-cache=/root/.cache/git" + - "--timeout=75" + - "--json" - "--clean" env: - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/scenarios/kubernetes_e2e.py b/scenarios/kubernetes_e2e.py index 5a86429bd99e..0d981a36aefe 100755 --- a/scenarios/kubernetes_e2e.py +++ b/scenarios/kubernetes_e2e.py @@ -63,6 +63,32 @@ def parse_env(env): """Returns (FOO, BAR=MORE) for FOO=BAR=MORE.""" return env.split('=', 1) +def kubeadm_version(mode): + """Return string to use for kubeadm version, given the job's mode (ci/pull/periodic).""" + version = '' + if mode in ['ci', 'periodic']: + # This job only runs against the kubernetes repo, and bootstrap.py leaves the + # current working directory at the repository root. Grab the SCM_REVISION so we + # can use the .debs built during the bazel-build job that should have already + # succeeded. + status = re.search( + r'STABLE_BUILD_SCM_REVISION ([^\n]+)', + check_output('hack/print-workspace-status.sh') + ) + if not status: + raise ValueError('STABLE_BUILD_SCM_REVISION not found') + version = status.group(1) + + elif mode == 'pull': + version = '%s/%s' % (os.environ['PULL_NUMBER'], os.getenv('PULL_REFS')) + + else: + raise ValueError("Unknown kubeadm mode given: %s" % mode) + + # The path given here should match jobs/ci-kubernetes-bazel-build.sh + return 'gs://kubernetes-release-dev/bazel/%s/bin/linux/amd64/' % version + + class LocalMode(object): """Runs e2e tests by calling e2e-runner.sh.""" def __init__(self, workspace): @@ -335,25 +361,12 @@ def main(args): if args.kubeadm: # Not from Jenkins cluster = args.cluster or 'e2e-kubeadm-%s' % os.getenv('BUILD_NUMBER', 0) - - # This job only runs against the kubernetes repo, and bootstrap.py leaves the - # current working directory at the repository root. Grab the SCM_REVISION so we - # can use the .debs built during the bazel-build job that should have already - # succeeded. - status = re.search( - r'STABLE_BUILD_SCM_REVISION ([^\n]+)', - check_output('hack/print-workspace-status.sh') - ) - if not status: - raise ValueError('STABLE_BUILD_SCM_REVISION not found') - + version = kubeadm_version(args.kubeadm) opt = '--deployment kubernetes-anywhere' \ ' --kubernetes-anywhere-path /workspace/kubernetes-anywhere' \ ' --kubernetes-anywhere-phase2-provider kubeadm' \ ' --kubernetes-anywhere-cluster %s' \ - ' --kubernetes-anywhere-kubeadm-version' \ - ' gs://kubernetes-release-dev/bazel/%s/build/debs/' % (cluster, status.group(1)) - # The gs:// path given here should match jobs/ci-kubernetes-bazel-build.sh + ' --kubernetes-anywhere-kubeadm-version %s' % (cluster, version) mode.add_environment('E2E_OPT=%s' % opt) # TODO(fejta): delete this? @@ -441,7 +454,7 @@ def create_parser(): parser.add_argument( '--down', default='true', help='If we need to set --down in e2e.go') parser.add_argument( - '--kubeadm', action='store_true', help='If the test is a kubeadm job') + '--kubeadm', choices=['ci', 'periodic', 'pull']) parser.add_argument( '--soak-test', action='store_true', help='If the test is a soak test job') parser.add_argument( diff --git a/scenarios/kubernetes_e2e_test.py b/scenarios/kubernetes_e2e_test.py index b651141f0519..a008bab5af54 100755 --- a/scenarios/kubernetes_e2e_test.py +++ b/scenarios/kubernetes_e2e_test.py @@ -20,6 +20,7 @@ """Test for kubernetes_e2e.py""" import json +import os import re import shutil import string @@ -111,17 +112,18 @@ def test_local(self): for call in self.callstack: self.assertFalse(call.startswith('docker')) - def test_kubeadm(self): - """Make sure kubeadm mode is fine overall.""" - args = self.parser.parse_args(['--mode=local', '--kubeadm']) + def test_kubeadm_ci(self): + """Make sure kubeadm ci mode is fine overall.""" + args = self.parser.parse_args(['--mode=local', '--kubeadm=ci']) self.assertEqual(args.mode, 'local') - self.assertEqual(args.kubeadm, True) + self.assertEqual(args.kubeadm, 'ci') with Stub(kubernetes_e2e, 'check_env', self.fake_check_env): with Stub(kubernetes_e2e, 'check_output', self.fake_output_work_status): kubernetes_e2e.main(args) self.assertIn('E2E_OPT', self.envs) - self.assertIn('v1.7.0-alpha.0.1320+599539dc0b9997', self.envs['E2E_OPT']) + self.assertIn('--kubernetes-anywhere-kubeadm-version gs://kubernetes-release-dev/bazel/' + 'v1.7.0-alpha.0.1320+599539dc0b9997/bin/linux/amd64/', self.envs['E2E_OPT']) called = False for call in self.callstack: self.assertFalse(call.startswith('docker')) @@ -129,6 +131,7 @@ def test_kubeadm(self): called = True self.assertTrue(called) + def test_include_host_env(self): """Ensure that host variables (such as GOPATH) are included.""" mode = kubernetes_e2e.LocalMode('/orig-workspace') @@ -138,6 +141,46 @@ def test_include_host_env(self): self.assertIn(['WORKSPACE', '/new/workspace'], mode.env) self.assertIn(['GOPATH', '/go/path'], mode.env) + def test_kubeadm_periodic(self): + """Make sure kubeadm periodic mode is fine overall.""" + args = self.parser.parse_args(['--mode=local', '--kubeadm=periodic']) + self.assertEqual(args.mode, 'local') + self.assertEqual(args.kubeadm, 'periodic') + with Stub(kubernetes_e2e, 'check_env', self.fake_check_env): + with Stub(kubernetes_e2e, 'check_output', self.fake_output_work_status): + kubernetes_e2e.main(args) + + self.assertIn('E2E_OPT', self.envs) + self.assertIn('--kubernetes-anywhere-kubeadm-version gs://kubernetes-release-dev/bazel/' + 'v1.7.0-alpha.0.1320+599539dc0b9997/bin/linux/amd64/', self.envs['E2E_OPT']) + called = False + for call in self.callstack: + self.assertFalse(call.startswith('docker')) + if call == 'hack/print-workspace-status.sh': + called = True + self.assertTrue(called) + + def test_kubeadm_pull(self): + """Make sure kubeadm pull mode is fine overall.""" + args = self.parser.parse_args(['--mode=local', '--kubeadm=pull']) + self.assertEqual(args.mode, 'local') + self.assertEqual(args.kubeadm, 'pull') + fake_env = {'PULL_NUMBER': 1234, 'PULL_REFS': 'master:abcd'} + with Stub(kubernetes_e2e, 'check_env', self.fake_check_env): + with Stub(os, 'environ', fake_env): + kubernetes_e2e.main(args) + + self.assertIn('E2E_OPT', self.envs) + self.assertIn('--kubernetes-anywhere-kubeadm-version gs://kubernetes-release-dev/bazel/' + '1234/master:abcd/bin/linux/amd64/', self.envs['E2E_OPT']) + + def test_kubeadm_invalid(self): + """Make sure kubeadm invalid mode exits unsuccessfully.""" + with self.assertRaises(SystemExit) as sysexit: + self.parser.parse_args(['--mode=local', '--kubeadm=deploy']) + + self.assertEqual(sysexit.exception.code, 2) + class DockerTest(ScenarioTest): """Class for testing e2e scenario in docker mode.""" def test_docker(self):