diff --git a/jobs/config.json b/jobs/config.json index 4217210e7853c..edc8560e86881 100644 --- a/jobs/config.json +++ b/jobs/config.json @@ -2108,7 +2108,7 @@ "--cluster=", "--env-file=platforms/gce.env", "--env-file=jobs/ci-kubernetes-e2e-kubeadm-gce.env", - "--kubeadm", + "--kubeadm=ci", "--mode=local" ], "scenario": "kubernetes_e2e" @@ -2118,7 +2118,7 @@ "--cluster=", "--env-file=platforms/gce.env", "--env-file=jobs/ci-kubernetes-e2e-kubeadm-gce-1-6.env", - "--kubeadm", + "--kubeadm=ci", "--mode=local" ], "scenario": "kubernetes_e2e" @@ -2667,7 +2667,7 @@ "--cluster=", "--env-file=platforms/gce.env", "--env-file=jobs/periodic-kubernetes-e2e-kubeadm-gce-1-6.env", - "--kubeadm", + "--kubeadm=periodic", "--mode=local" ], "scenario": "kubernetes_e2e" @@ -2764,6 +2764,16 @@ ], "scenario": "kubernetes_e2e" }, + "pull-kubernetes-e2e-kubeadm-gce": { + "args": [ + "--cluster=", + "--env-file=platforms/gce.env", + "--env-file=jobs/pull-kubernetes-e2e-kubeadm-gce.env", + "--kubeadm=pull", + "--mode=local" + ], + "scenario": "kubernetes_e2e" + }, "pull-kubernetes-federation-e2e-gce": { "args": [ "--env-file=platforms/gce.env", @@ -2820,4 +2830,4 @@ ], "scenario": "kubernetes_verify" } -} \ No newline at end of file +} diff --git a/jobs/pull-kubernetes-e2e-kubeadm-gce.env b/jobs/pull-kubernetes-e2e-kubeadm-gce.env new file mode 100755 index 0000000000000..1c3b690b5f8c5 --- /dev/null +++ b/jobs/pull-kubernetes-e2e-kubeadm-gce.env @@ -0,0 +1,15 @@ +### job-env + +PROJECT=k8s-jkns-pr-kubeadm +KUBERNETES_PROVIDER=kubernetes-anywhere + +GINKGO_PARALLEL=y +GINKGO_TEST_ARGS='--ginkgo.focus=\[Conformance\] --ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]' + +# Resource leak detection is disabled because prow runs multiple instances of +# this job in the same project concurrently, and resource leak detection will +# make the job flaky. +FAIL_ON_GCP_RESOURCE_LEAK=false + +# After post-env +KUBEKINS_TIMEOUT=120m diff --git a/jobs/pull-kubernetes-e2e-kubeadm-gce.sh b/jobs/pull-kubernetes-e2e-kubeadm-gce.sh deleted file mode 100755 index ce13ad66e3d81..0000000000000 --- a/jobs/pull-kubernetes-e2e-kubeadm-gce.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -# Copyright 2016 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail -set -o xtrace - -readonly testinfra="$(dirname "${0}")/.." - -### builder - -### job-env - -export PROJECT="k8s-jkns-pr-kubeadm" -export KUBERNETES_PROVIDER=kubernetes-anywhere -# These deliberately do not use the :- syntax so that the nounset flag produces -# succinct errors if either is unset. This job depends on the artifacts created -# during pull-kubernetes-bazel, so these env vars must be set correctly to find -# this pull's build artifacts. -export VERSION=${PULL_NUMBER}/${PULL_REFS} - -export E2E_NAME="e2e-kubeadm-${BUILD_NUMBER:-0}" -export E2E_OPT="--deployment kubernetes-anywhere --kubernetes-anywhere-path /workspace/kubernetes-anywhere" -export E2E_OPT+=" --kubernetes-anywhere-phase2-provider kubeadm --kubernetes-anywhere-cluster ${E2E_NAME}" -# The gs:// path given here should match jobs/pull-kubernetes-bazel.sh -export E2E_OPT+=" --kubernetes-anywhere-kubeadm-version gs://kubernetes-release-dev/bazel/${VERSION}/build/debs/" -export GINKGO_TEST_ARGS="--ginkgo.focus=\[Conformance\]" - -# Resource leak detection is disabled because prow runs multiple instances of -# this job in the same project concurrently, and resource leak detection will -# make the job flaky. -export FAIL_ON_GCP_RESOURCE_LEAK="false" - -### post-env - -# Assume we're upping, testing, and downing a cluster -export E2E_UP="${E2E_UP:-true}" -# TODO(pipejakob): Reenable testing when we have a pod network that works with -# kubeadm 1.6 clusters. For now, simply bringing up a cluster is a good e2e -# test, since it will only succeed if the apiserver is healthy and all -# expected nodes are registered. -export E2E_TEST="false" -export E2E_DOWN="${E2E_DOWN:-true}" - -# Skip gcloud update checking -export CLOUDSDK_COMPONENT_MANAGER_DISABLE_UPDATE_CHECK=true -# Use default component update behavior -export CLOUDSDK_EXPERIMENTAL_FAST_COMPONENT_UPDATE=false - -# Get golang into our PATH so we can run e2e.go -export PATH="${PATH}:/usr/local/go/bin" - -# After post-env -export GINKGO_PARALLEL="y" - -### Runner -readonly runner="/workspace/e2e-runner.sh" -export DOCKER_TIMEOUT="140m" -export KUBEKINS_TIMEOUT="120m" -"${runner}" diff --git a/scenarios/kubernetes_e2e.py b/scenarios/kubernetes_e2e.py index 670d56c7048b6..e458acce6775b 100755 --- a/scenarios/kubernetes_e2e.py +++ b/scenarios/kubernetes_e2e.py @@ -60,6 +60,31 @@ def kubekins(tag): return 'gcr.io/k8s-testimages/kubekins-e2e:%s' % tag +def kubeadm_version(mode): + version = '' + if mode in ['ci', 'periodic']: + # This job only runs against the kubernetes repo, and bootstrap.py leaves the + # current working directory at the repository root. Grab the SCM_REVISION so we + # can use the .debs built during the bazel-build job that should have already + # succeeded. + status = re.search( + r'STABLE_BUILD_SCM_REVISION ([^\n]+)', + check_output('hack/print-workspace-status.sh') + ) + if not status: + raise ValueError('STABLE_BUILD_SCM_REVISION not found') + version = status.group(1) + + elif mode == 'pull': + version = '%s/%s' % (os.environ['PULL_NUMBER'], os.getenv('PULL_REFS')) + + else: + raise ValueError("Unknown kubeadm mode given: %s" % mode) + + # The path given here should match jobs/ci-kubernetes-bazel-build.sh + return 'gs://kubernetes-release-dev/bazel/%s/bin/linux/amd64/' % version + + class LocalMode(object): """Runs e2e tests by calling e2e-runner.sh.""" def __init__(self, workspace): @@ -313,29 +338,16 @@ def main(args): cluster = args.cluster or 'e2e-gce-%s-%s' % ( os.environ['NODE_NAME'], os.getenv('EXECUTOR_NUMBER', 0)) - if args.kubeadm: - # Not from Jenkins - cluster = args.cluster or 'e2e-kubeadm-%s' % os.getenv('BUILD_NUMBER', 0) - - # This job only runs against the kubernetes repo, and bootstrap.py leaves the - # current working directory at the repository root. Grab the SCM_REVISION so we - # can use the .debs built during the bazel-build job that should have already - # succeeded. - status = re.search( - r'STABLE_BUILD_SCM_REVISION ([^\n]+)', - check_output('hack/print-workspace-status.sh') - ) - if not status: - raise ValueError('STABLE_BUILD_SCM_REVISION not found') - - opt = '--deployment kubernetes-anywhere' \ - ' --kubernetes-anywhere-path /workspace/kubernetes-anywhere' \ - ' --kubernetes-anywhere-phase2-provider kubeadm' \ - ' --kubernetes-anywhere-cluster %s' \ - ' --kubernetes-anywhere-kubeadm-version' \ - ' gs://kubernetes-release-dev/bazel/%s/build/debs/' % (cluster, status.group(1)) - # The gs:// path given here should match jobs/ci-kubernetes-bazel-build.sh - mode.add_environment('E2E_OPT=%s' % opt) + if args.kubeadm is not None: + # Not from Jenkins + cluster = args.cluster or 'e2e-kubeadm-%s' % os.getenv('BUILD_NUMBER', 0) + version = kubeadm_version(args.kubeadm) + opt = '--deployment kubernetes-anywhere' \ + ' --kubernetes-anywhere-path /workspace/kubernetes-anywhere' \ + ' --kubernetes-anywhere-phase2-provider kubeadm' \ + ' --kubernetes-anywhere-cluster %s' \ + ' --kubernetes-anywhere-kubeadm-version %s' % (cluster, version) + mode.add_environment('E2E_OPT=%s' % opt) # env blacklist. # TODO(krzyzacy) change this to a whitelist @@ -435,7 +447,7 @@ def create_parser(): parser.add_argument( '--down', default='true', help='If we need to set --down in e2e.go') parser.add_argument( - '--kubeadm', action='store_true', help='If the test is a kubeadm job') + '--kubeadm', choices=['ci', 'periodic', 'pull']) parser.add_argument( '--soak-test', action='store_true', help='If the test is a soak test job') parser.add_argument( diff --git a/scenarios/kubernetes_e2e_test.py b/scenarios/kubernetes_e2e_test.py index efe43a0d45345..684eeecf33fb0 100755 --- a/scenarios/kubernetes_e2e_test.py +++ b/scenarios/kubernetes_e2e_test.py @@ -20,6 +20,7 @@ """Test for kubernetes_e2e.py""" import json +import os import re import shutil import string @@ -111,17 +112,17 @@ def test_local(self): for call in self.callstack: self.assertFalse(call.startswith('docker')) - def test_kubeadm(self): - """Make sure kubeadm mode is fine overall.""" - args = self.parser.parse_args(['--mode=local', '--kubeadm']) + def test_kubeadm_ci(self): + """Make sure kubeadm ci mode is fine overall.""" + args = self.parser.parse_args(['--mode=local', '--kubeadm=ci']) self.assertEqual(args.mode, 'local') - self.assertEqual(args.kubeadm, True) + self.assertEqual(args.kubeadm, 'ci') with Stub(kubernetes_e2e, 'check_env', self.fake_check_env): with Stub(kubernetes_e2e, 'check_output', self.fake_output_work_status): kubernetes_e2e.main(args) self.assertIn('E2E_OPT', self.envs) - self.assertIn('v1.7.0-alpha.0.1320+599539dc0b9997', self.envs['E2E_OPT']) + self.assertIn('--kubernetes-anywhere-kubeadm-version gs://kubernetes-release-dev/bazel/v1.7.0-alpha.0.1320+599539dc0b9997/bin/linux/amd64/', self.envs['E2E_OPT']) called = False for call in self.callstack: self.assertFalse(call.startswith('docker')) @@ -129,6 +130,44 @@ def test_kubeadm(self): called = True self.assertTrue(called) + def test_kubeadm_periodic(self): + """Make sure kubeadm periodic mode is fine overall.""" + args = self.parser.parse_args(['--mode=local', '--kubeadm=periodic']) + self.assertEqual(args.mode, 'local') + self.assertEqual(args.kubeadm, 'periodic') + with Stub(kubernetes_e2e, 'check_env', self.fake_check_env): + with Stub(kubernetes_e2e, 'check_output', self.fake_output_work_status): + kubernetes_e2e.main(args) + + self.assertIn('E2E_OPT', self.envs) + self.assertIn('--kubernetes-anywhere-kubeadm-version gs://kubernetes-release-dev/bazel/v1.7.0-alpha.0.1320+599539dc0b9997/bin/linux/amd64/', self.envs['E2E_OPT']) + called = False + for call in self.callstack: + self.assertFalse(call.startswith('docker')) + if call == 'hack/print-workspace-status.sh': + called = True + self.assertTrue(called) + + def test_kubeadm_pull(self): + """Make sure kubeadm pull mode is fine overall.""" + args = self.parser.parse_args(['--mode=local', '--kubeadm=pull']) + self.assertEqual(args.mode, 'local') + self.assertEqual(args.kubeadm, 'pull') + fake_env = {'PULL_NUMBER': 1234, 'PULL_REFS': 'master:abcd'} + with Stub(kubernetes_e2e, 'check_env', self.fake_check_env): + with Stub(os, 'environ', fake_env): + kubernetes_e2e.main(args) + + self.assertIn('E2E_OPT', self.envs) + self.assertIn('--kubernetes-anywhere-kubeadm-version gs://kubernetes-release-dev/bazel/1234/master:abcd/bin/linux/amd64/', self.envs['E2E_OPT']) + + def test_kubeadm_invalid(self): + """Make sure kubeadm invalid mode exits unsuccessfully.""" + with self.assertRaises(SystemExit) as sysexit: + self.parser.parse_args(['--mode=local', '--kubeadm=deploy']) + + self.assertEqual(sysexit.exception.code, 2) + class DockerTest(ScenarioTest): """Class for testing e2e scenario in docker mode.""" def test_docker(self):