From 1b011d1d104293c68ab9ac7e8fcdffd5f22a6cc6 Mon Sep 17 00:00:00 2001 From: Jacob Beacham Date: Fri, 5 May 2017 11:53:12 -0700 Subject: [PATCH] Convert kubeadm pull job to scenario. This also changes the --kubeadm boolean flag in kubernetes_e2e.py to an option to specify the type of job (ci, pull, periodic) so that it knows where to find the .deb artifacts created by the previous bazel job, since the output location can differ by type. --- jobs/config.json | 18 ++++-- jobs/pull-kubernetes-e2e-kubeadm-gce.env | 15 +++++ jobs/pull-kubernetes-e2e-kubeadm-gce.sh | 73 ------------------------ scenarios/kubernetes_e2e.py | 60 +++++++++++-------- scenarios/kubernetes_e2e_test.py | 49 ++++++++++++++-- 5 files changed, 109 insertions(+), 106 deletions(-) create mode 100755 jobs/pull-kubernetes-e2e-kubeadm-gce.env delete mode 100755 jobs/pull-kubernetes-e2e-kubeadm-gce.sh diff --git a/jobs/config.json b/jobs/config.json index 4217210e7853c..edc8560e86881 100644 --- a/jobs/config.json +++ b/jobs/config.json @@ -2108,7 +2108,7 @@ "--cluster=", "--env-file=platforms/gce.env", "--env-file=jobs/ci-kubernetes-e2e-kubeadm-gce.env", - "--kubeadm", + "--kubeadm=ci", "--mode=local" ], "scenario": "kubernetes_e2e" @@ -2118,7 +2118,7 @@ "--cluster=", "--env-file=platforms/gce.env", "--env-file=jobs/ci-kubernetes-e2e-kubeadm-gce-1-6.env", - "--kubeadm", + "--kubeadm=ci", "--mode=local" ], "scenario": "kubernetes_e2e" @@ -2667,7 +2667,7 @@ "--cluster=", "--env-file=platforms/gce.env", "--env-file=jobs/periodic-kubernetes-e2e-kubeadm-gce-1-6.env", - "--kubeadm", + "--kubeadm=periodic", "--mode=local" ], "scenario": "kubernetes_e2e" @@ -2764,6 +2764,16 @@ ], "scenario": "kubernetes_e2e" }, + "pull-kubernetes-e2e-kubeadm-gce": { + "args": [ + "--cluster=", + "--env-file=platforms/gce.env", + "--env-file=jobs/pull-kubernetes-e2e-kubeadm-gce.env", + "--kubeadm=pull", + "--mode=local" + ], + "scenario": "kubernetes_e2e" + }, "pull-kubernetes-federation-e2e-gce": { "args": [ "--env-file=platforms/gce.env", @@ -2820,4 +2830,4 @@ ], "scenario": "kubernetes_verify" } -} \ No newline at end of file +} diff --git a/jobs/pull-kubernetes-e2e-kubeadm-gce.env b/jobs/pull-kubernetes-e2e-kubeadm-gce.env new file mode 100755 index 0000000000000..1c3b690b5f8c5 --- /dev/null +++ b/jobs/pull-kubernetes-e2e-kubeadm-gce.env @@ -0,0 +1,15 @@ +### job-env + +PROJECT=k8s-jkns-pr-kubeadm +KUBERNETES_PROVIDER=kubernetes-anywhere + +GINKGO_PARALLEL=y +GINKGO_TEST_ARGS='--ginkgo.focus=\[Conformance\] --ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]' + +# Resource leak detection is disabled because prow runs multiple instances of +# this job in the same project concurrently, and resource leak detection will +# make the job flaky. +FAIL_ON_GCP_RESOURCE_LEAK=false + +# After post-env +KUBEKINS_TIMEOUT=120m diff --git a/jobs/pull-kubernetes-e2e-kubeadm-gce.sh b/jobs/pull-kubernetes-e2e-kubeadm-gce.sh deleted file mode 100755 index ce13ad66e3d81..0000000000000 --- a/jobs/pull-kubernetes-e2e-kubeadm-gce.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -# Copyright 2016 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail -set -o xtrace - -readonly testinfra="$(dirname "${0}")/.." - -### builder - -### job-env - -export PROJECT="k8s-jkns-pr-kubeadm" -export KUBERNETES_PROVIDER=kubernetes-anywhere -# These deliberately do not use the :- syntax so that the nounset flag produces -# succinct errors if either is unset. This job depends on the artifacts created -# during pull-kubernetes-bazel, so these env vars must be set correctly to find -# this pull's build artifacts. -export VERSION=${PULL_NUMBER}/${PULL_REFS} - -export E2E_NAME="e2e-kubeadm-${BUILD_NUMBER:-0}" -export E2E_OPT="--deployment kubernetes-anywhere --kubernetes-anywhere-path /workspace/kubernetes-anywhere" -export E2E_OPT+=" --kubernetes-anywhere-phase2-provider kubeadm --kubernetes-anywhere-cluster ${E2E_NAME}" -# The gs:// path given here should match jobs/pull-kubernetes-bazel.sh -export E2E_OPT+=" --kubernetes-anywhere-kubeadm-version gs://kubernetes-release-dev/bazel/${VERSION}/build/debs/" -export GINKGO_TEST_ARGS="--ginkgo.focus=\[Conformance\]" - -# Resource leak detection is disabled because prow runs multiple instances of -# this job in the same project concurrently, and resource leak detection will -# make the job flaky. -export FAIL_ON_GCP_RESOURCE_LEAK="false" - -### post-env - -# Assume we're upping, testing, and downing a cluster -export E2E_UP="${E2E_UP:-true}" -# TODO(pipejakob): Reenable testing when we have a pod network that works with -# kubeadm 1.6 clusters. For now, simply bringing up a cluster is a good e2e -# test, since it will only succeed if the apiserver is healthy and all -# expected nodes are registered. -export E2E_TEST="false" -export E2E_DOWN="${E2E_DOWN:-true}" - -# Skip gcloud update checking -export CLOUDSDK_COMPONENT_MANAGER_DISABLE_UPDATE_CHECK=true -# Use default component update behavior -export CLOUDSDK_EXPERIMENTAL_FAST_COMPONENT_UPDATE=false - -# Get golang into our PATH so we can run e2e.go -export PATH="${PATH}:/usr/local/go/bin" - -# After post-env -export GINKGO_PARALLEL="y" - -### Runner -readonly runner="/workspace/e2e-runner.sh" -export DOCKER_TIMEOUT="140m" -export KUBEKINS_TIMEOUT="120m" -"${runner}" diff --git a/scenarios/kubernetes_e2e.py b/scenarios/kubernetes_e2e.py index 670d56c7048b6..e458acce6775b 100755 --- a/scenarios/kubernetes_e2e.py +++ b/scenarios/kubernetes_e2e.py @@ -60,6 +60,31 @@ def kubekins(tag): return 'gcr.io/k8s-testimages/kubekins-e2e:%s' % tag +def kubeadm_version(mode): + version = '' + if mode in ['ci', 'periodic']: + # This job only runs against the kubernetes repo, and bootstrap.py leaves the + # current working directory at the repository root. Grab the SCM_REVISION so we + # can use the .debs built during the bazel-build job that should have already + # succeeded. + status = re.search( + r'STABLE_BUILD_SCM_REVISION ([^\n]+)', + check_output('hack/print-workspace-status.sh') + ) + if not status: + raise ValueError('STABLE_BUILD_SCM_REVISION not found') + version = status.group(1) + + elif mode == 'pull': + version = '%s/%s' % (os.environ['PULL_NUMBER'], os.getenv('PULL_REFS')) + + else: + raise ValueError("Unknown kubeadm mode given: %s" % mode) + + # The path given here should match jobs/ci-kubernetes-bazel-build.sh + return 'gs://kubernetes-release-dev/bazel/%s/bin/linux/amd64/' % version + + class LocalMode(object): """Runs e2e tests by calling e2e-runner.sh.""" def __init__(self, workspace): @@ -313,29 +338,16 @@ def main(args): cluster = args.cluster or 'e2e-gce-%s-%s' % ( os.environ['NODE_NAME'], os.getenv('EXECUTOR_NUMBER', 0)) - if args.kubeadm: - # Not from Jenkins - cluster = args.cluster or 'e2e-kubeadm-%s' % os.getenv('BUILD_NUMBER', 0) - - # This job only runs against the kubernetes repo, and bootstrap.py leaves the - # current working directory at the repository root. Grab the SCM_REVISION so we - # can use the .debs built during the bazel-build job that should have already - # succeeded. - status = re.search( - r'STABLE_BUILD_SCM_REVISION ([^\n]+)', - check_output('hack/print-workspace-status.sh') - ) - if not status: - raise ValueError('STABLE_BUILD_SCM_REVISION not found') - - opt = '--deployment kubernetes-anywhere' \ - ' --kubernetes-anywhere-path /workspace/kubernetes-anywhere' \ - ' --kubernetes-anywhere-phase2-provider kubeadm' \ - ' --kubernetes-anywhere-cluster %s' \ - ' --kubernetes-anywhere-kubeadm-version' \ - ' gs://kubernetes-release-dev/bazel/%s/build/debs/' % (cluster, status.group(1)) - # The gs:// path given here should match jobs/ci-kubernetes-bazel-build.sh - mode.add_environment('E2E_OPT=%s' % opt) + if args.kubeadm is not None: + # Not from Jenkins + cluster = args.cluster or 'e2e-kubeadm-%s' % os.getenv('BUILD_NUMBER', 0) + version = kubeadm_version(args.kubeadm) + opt = '--deployment kubernetes-anywhere' \ + ' --kubernetes-anywhere-path /workspace/kubernetes-anywhere' \ + ' --kubernetes-anywhere-phase2-provider kubeadm' \ + ' --kubernetes-anywhere-cluster %s' \ + ' --kubernetes-anywhere-kubeadm-version %s' % (cluster, version) + mode.add_environment('E2E_OPT=%s' % opt) # env blacklist. # TODO(krzyzacy) change this to a whitelist @@ -435,7 +447,7 @@ def create_parser(): parser.add_argument( '--down', default='true', help='If we need to set --down in e2e.go') parser.add_argument( - '--kubeadm', action='store_true', help='If the test is a kubeadm job') + '--kubeadm', choices=['ci', 'periodic', 'pull']) parser.add_argument( '--soak-test', action='store_true', help='If the test is a soak test job') parser.add_argument( diff --git a/scenarios/kubernetes_e2e_test.py b/scenarios/kubernetes_e2e_test.py index efe43a0d45345..684eeecf33fb0 100755 --- a/scenarios/kubernetes_e2e_test.py +++ b/scenarios/kubernetes_e2e_test.py @@ -20,6 +20,7 @@ """Test for kubernetes_e2e.py""" import json +import os import re import shutil import string @@ -111,17 +112,17 @@ def test_local(self): for call in self.callstack: self.assertFalse(call.startswith('docker')) - def test_kubeadm(self): - """Make sure kubeadm mode is fine overall.""" - args = self.parser.parse_args(['--mode=local', '--kubeadm']) + def test_kubeadm_ci(self): + """Make sure kubeadm ci mode is fine overall.""" + args = self.parser.parse_args(['--mode=local', '--kubeadm=ci']) self.assertEqual(args.mode, 'local') - self.assertEqual(args.kubeadm, True) + self.assertEqual(args.kubeadm, 'ci') with Stub(kubernetes_e2e, 'check_env', self.fake_check_env): with Stub(kubernetes_e2e, 'check_output', self.fake_output_work_status): kubernetes_e2e.main(args) self.assertIn('E2E_OPT', self.envs) - self.assertIn('v1.7.0-alpha.0.1320+599539dc0b9997', self.envs['E2E_OPT']) + self.assertIn('--kubernetes-anywhere-kubeadm-version gs://kubernetes-release-dev/bazel/v1.7.0-alpha.0.1320+599539dc0b9997/bin/linux/amd64/', self.envs['E2E_OPT']) called = False for call in self.callstack: self.assertFalse(call.startswith('docker')) @@ -129,6 +130,44 @@ def test_kubeadm(self): called = True self.assertTrue(called) + def test_kubeadm_periodic(self): + """Make sure kubeadm periodic mode is fine overall.""" + args = self.parser.parse_args(['--mode=local', '--kubeadm=periodic']) + self.assertEqual(args.mode, 'local') + self.assertEqual(args.kubeadm, 'periodic') + with Stub(kubernetes_e2e, 'check_env', self.fake_check_env): + with Stub(kubernetes_e2e, 'check_output', self.fake_output_work_status): + kubernetes_e2e.main(args) + + self.assertIn('E2E_OPT', self.envs) + self.assertIn('--kubernetes-anywhere-kubeadm-version gs://kubernetes-release-dev/bazel/v1.7.0-alpha.0.1320+599539dc0b9997/bin/linux/amd64/', self.envs['E2E_OPT']) + called = False + for call in self.callstack: + self.assertFalse(call.startswith('docker')) + if call == 'hack/print-workspace-status.sh': + called = True + self.assertTrue(called) + + def test_kubeadm_pull(self): + """Make sure kubeadm pull mode is fine overall.""" + args = self.parser.parse_args(['--mode=local', '--kubeadm=pull']) + self.assertEqual(args.mode, 'local') + self.assertEqual(args.kubeadm, 'pull') + fake_env = {'PULL_NUMBER': 1234, 'PULL_REFS': 'master:abcd'} + with Stub(kubernetes_e2e, 'check_env', self.fake_check_env): + with Stub(os, 'environ', fake_env): + kubernetes_e2e.main(args) + + self.assertIn('E2E_OPT', self.envs) + self.assertIn('--kubernetes-anywhere-kubeadm-version gs://kubernetes-release-dev/bazel/1234/master:abcd/bin/linux/amd64/', self.envs['E2E_OPT']) + + def test_kubeadm_invalid(self): + """Make sure kubeadm invalid mode exits unsuccessfully.""" + with self.assertRaises(SystemExit) as sysexit: + self.parser.parse_args(['--mode=local', '--kubeadm=deploy']) + + self.assertEqual(sysexit.exception.code, 2) + class DockerTest(ScenarioTest): """Class for testing e2e scenario in docker mode.""" def test_docker(self):