diff --git a/.travis.yml b/.travis.yml index 5bd750453a563..dd493363ab1e7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,23 +26,24 @@ env: - TRAVIS_CACHE=$HOME/.travis_cache/ matrix: - TOX_ENV=flake8 - - TOX_ENV=py27-backend_mysql - - TOX_ENV=py27-backend_sqlite - - TOX_ENV=py27-backend_postgres - - TOX_ENV=py35-backend_mysql PYTHON_VERSION=3 - - TOX_ENV=py35-backend_sqlite PYTHON_VERSION=3 - - TOX_ENV=py35-backend_postgres PYTHON_VERSION=3 - - TOX_ENV=py27-backend_postgres KUBERNETES_VERSION=v1.9.0 - - TOX_ENV=py35-backend_postgres KUBERNETES_VERSION=v1.10.0 PYTHON_VERSION=3 + - TOX_ENV=py27-backend_mysql-env_docker + - TOX_ENV=py27-backend_sqlite-env_docker + - TOX_ENV=py27-backend_postgres-env_docker + - TOX_ENV=py35-backend_mysql-env_docker PYTHON_VERSION=3 + - TOX_ENV=py35-backend_sqlite-env_docker PYTHON_VERSION=3 + - TOX_ENV=py35-backend_postgres-env_docker PYTHON_VERSION=3 + - TOX_ENV=py27-backend_postgres-env_kubernetes KUBERNETES_VERSION=v1.9.0 + - TOX_ENV=py35-backend_postgres-env_kubernetes KUBERNETES_VERSION=v1.10.0 PYTHON_VERSION=3 + cache: directories: - $HOME/.wheelhouse/ - $HOME/.cache/pip - $HOME/.travis_cache/ before_install: - - sudo ls -lh $HOME/.cache/pip/ - - sudo rm -rf $HOME/.cache/pip/* $HOME/.wheelhouse/* - - sudo chown -R travis:travis $HOME/.cache/pip + # Required for K8s v1.10.x. See + # https://github.com/kubernetes/kubernetes/issues/61058#issuecomment-372764783 + - if [ ! -z "$KUBERNETES_VERSION" ]; then sudo mount --make-shared / && sudo service docker restart; fi install: # Use recent docker-compose version - sudo rm /usr/local/bin/docker-compose @@ -50,5 +51,12 @@ install: - chmod +x docker-compose - sudo mv docker-compose /usr/local/bin - pip install --upgrade pip + - if [ ! -z "$KUBERNETES_VERSION" ]; then ./scripts/ci/kubernetes/setup_kubernetes.sh; fi script: - - docker-compose --log-level ERROR -f scripts/ci/docker-compose.yml run airflow-testing /app/scripts/ci/run-ci.sh + - if [ -z "$KUBERNETES_VERSION" ]; then docker-compose --log-level ERROR -f scripts/ci/docker-compose.yml run airflow-testing /app/scripts/ci/run-ci.sh; fi + - if [ ! -z "$KUBERNETES_VERSION" ]; then + ./scripts/ci/kubernetes/kube/deploy.sh && + MINIKUBE_IP=$(minikube ip) docker-compose --log-level ERROR -f scripts/ci/docker-compose.yml -f scripts/ci/docker-compose-kubernetes.yml run airflow-testing /app/scripts/ci/run-ci.sh; + fi +before_cache: + - sudo chown -R travis:travis $HOME/.cache/pip $HOME/.wheelhouse/ diff --git a/airflow/example_dags_kubernetes/__init__.py b/airflow/example_dags_kubernetes/__init__.py new file mode 100644 index 0000000000000..114d189da14ab --- /dev/null +++ b/airflow/example_dags_kubernetes/__init__.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/example_dags_kubernetes/example_kubernetes_annotation.py b/airflow/example_dags_kubernetes/example_kubernetes_annotation.py new file mode 100644 index 0000000000000..058baf69901f7 --- /dev/null +++ b/airflow/example_dags_kubernetes/example_kubernetes_annotation.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import print_function +import airflow +from airflow.operators.python_operator import PythonOperator +from airflow.models import DAG + +args = { + 'owner': 'airflow', + 'start_date': airflow.utils.dates.days_ago(2) +} + +dag = DAG( + dag_id='example_kubernetes_annotation', default_args=args, + schedule_interval=None +) + + +def print_stuff(): + print("annotated!") + + +# You can use annotations on your kubernetes pods! +start_task = PythonOperator( + task_id="start_task", python_callable=print_stuff, dag=dag, + executor_config={ + "KubernetesExecutor": { + "annotations": {"test": "annotation"} + } + } +) diff --git a/scripts/ci/5-run-tests.sh b/scripts/ci/5-run-tests.sh index 6c6f53d4670e7..8acc78f1a7ff0 100755 --- a/scripts/ci/5-run-tests.sh +++ b/scripts/ci/5-run-tests.sh @@ -58,9 +58,11 @@ which airflow > /dev/null || python setup.py develop # (which contains /usr/local/bin) sudo ln -sf "${VIRTUAL_ENV}/bin/airflow" /usr/local/bin/ -echo "Initializing the DB" -yes | airflow initdb -yes | airflow resetdb +if [ -z "$KUBERNETES_VERSION" ]; then + echo "Initializing the DB" + yes | airflow initdb + yes | airflow resetdb +fi if [ -z "$nose_args" ]; then nose_args="--with-coverage \ @@ -75,8 +77,10 @@ if [ -z "$nose_args" ]; then --logging-level=DEBUG" fi -# kdc init happens in setup_kdc.sh -kinit -kt ${KRB5_KTNAME} airflow +if [ -z "$KUBERNETES_VERSION" ]; then + # kdc init happens in setup_kdc.sh + kinit -kt ${KRB5_KTNAME} airflow +fi # For impersonation tests running on SQLite on Travis, make the database world readable so other # users can update it diff --git a/scripts/ci/docker-compose-kubernetes.yml b/scripts/ci/docker-compose-kubernetes.yml new file mode 100644 index 0000000000000..0cdb2df199b7b --- /dev/null +++ b/scripts/ci/docker-compose-kubernetes.yml @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "2.2" +services: + airflow-testing: + network_mode: host + environment: + - KUBERNETES_VERSION + - MINIKUBE_IP + volumes: + - /usr/local/bin/kubectl:/usr/local/bin/kubectl + - /usr/local/bin/minikube:/usr/local/bin/minikube + - ~/.kube:/home/airflow/.kube + - ~/.minikube:/home/airflow/.minikube diff --git a/scripts/ci/kubernetes/docker/airflow-test-env-init.sh b/scripts/ci/kubernetes/docker/airflow-test-env-init.sh index aa86da72dfa70..33593fd1cac46 100755 --- a/scripts/ci/kubernetes/docker/airflow-test-env-init.sh +++ b/scripts/ci/kubernetes/docker/airflow-test-env-init.sh @@ -18,7 +18,7 @@ # under the License. cd /usr/local/lib/python2.7/dist-packages/airflow && \ -cp -R example_dags/* /root/airflow/dags/ && \ +cp -R example_dags_kubernetes/* /root/airflow/dags/ && \ airflow initdb && \ alembic upgrade heads && \ (airflow create_user -u airflow -l airflow -f jon -e airflow@apache.org -r Admin -p airflow || true) && \ diff --git a/scripts/ci/kubernetes/docker/build.sh b/scripts/ci/kubernetes/docker/build.sh index b93c6b1715ddb..49d0fb87c3a31 100755 --- a/scripts/ci/kubernetes/docker/build.sh +++ b/scripts/ci/kubernetes/docker/build.sh @@ -22,9 +22,13 @@ TAG=${2:-latest} DIRNAME=$(cd "$(dirname "$0")"; pwd) AIRFLOW_ROOT="$DIRNAME/../../../.." -ENVCONFIG=$(minikube docker-env) -if [ $? -eq 0 ]; then - eval $ENVCONFIG +set -e + +if [ "${VM_DRIVER:-none}" != "none" ]; then + ENVCONFIG=$(minikube docker-env) + if [ $? -eq 0 ]; then + eval $ENVCONFIG + fi fi echo "Airflow directory $AIRFLOW_ROOT" diff --git a/scripts/ci/kubernetes/kube/airflow.yaml b/scripts/ci/kubernetes/kube/airflow.yaml index 4f451ba44a687..1b6e33cec5b3c 100644 --- a/scripts/ci/kubernetes/kube/airflow.yaml +++ b/scripts/ci/kubernetes/kube/airflow.yaml @@ -151,4 +151,3 @@ spec: nodePort: 30809 selector: name: airflow - diff --git a/scripts/ci/kubernetes/kube/configmaps.yaml b/scripts/ci/kubernetes/kube/configmaps.yaml index c0c7e9b2d03e0..ab44931e59cc5 100644 --- a/scripts/ci/kubernetes/kube/configmaps.yaml +++ b/scripts/ci/kubernetes/kube/configmaps.yaml @@ -27,7 +27,7 @@ data: logging_level = INFO executor = KubernetesExecutor parallelism = 32 - load_examples = True + load_examples = False plugins_folder = /root/airflow/plugins sql_alchemy_conn = $SQL_ALCHEMY_CONN diff --git a/scripts/ci/kubernetes/kube/deploy.sh b/scripts/ci/kubernetes/kube/deploy.sh index a9a42a7a12d12..75567c9acf6a4 100755 --- a/scripts/ci/kubernetes/kube/deploy.sh +++ b/scripts/ci/kubernetes/kube/deploy.sh @@ -17,6 +17,8 @@ # specific language governing permissions and limitations * # under the License. * +set -x + IMAGE=${1:-airflow/ci} TAG=${2:-latest} DIRNAME=$(cd "$(dirname "$0")"; pwd) diff --git a/scripts/ci/kubernetes/minikube/start_minikube.sh b/scripts/ci/kubernetes/minikube/start_minikube.sh index c9388e08ffddc..0803884899bf5 100755 --- a/scripts/ci/kubernetes/minikube/start_minikube.sh +++ b/scripts/ci/kubernetes/minikube/start_minikube.sh @@ -19,12 +19,13 @@ #!/usr/bin/env bash +set -x _MY_SCRIPT="${BASH_SOURCE[0]}" _MY_DIR=$(cd "$(dirname "$_MY_SCRIPT")" && pwd) # Avoids 1.7.x because of https://github.com/kubernetes/minikube/issues/2240 _KUBERNETES_VERSION="${KUBERNETES_VERSION}" -_MINIKUBE_VERSION="${MINIKUBE_VERSION:-v0.26.0}" +_MINIKUBE_VERSION="${MINIKUBE_VERSION:-v0.28.2}" echo "setting up kubernetes ${_KUBERNETES_VERSION}, using minikube ${_MINIKUBE_VERSION}" @@ -116,7 +117,7 @@ echo Showing kube-system pods kubectl get -n kube-system pods (k8s_single_pod_ready -n kube-system -l component=kube-addon-manager) || - (_ADDON=$(kubectl get pod -n kube-system -l component=kube-addon-manager + (_ADDON=$(kubectl get pod -n kube-system -l component=kube-addon-manager \ --no-headers -o name| cut -d/ -f2); echo Addon-manager describe:; kubectl describe pod -n kube-system $_ADDON; diff --git a/scripts/ci/kubernetes/setup_kubernetes.sh b/scripts/ci/kubernetes/setup_kubernetes.sh index ea559a02aa997..96f8f5d6af2a5 100755 --- a/scripts/ci/kubernetes/setup_kubernetes.sh +++ b/scripts/ci/kubernetes/setup_kubernetes.sh @@ -25,6 +25,5 @@ DIRNAME=$(cd "$(dirname "$0")"; pwd) $DIRNAME/minikube/start_minikube.sh $DIRNAME/docker/build.sh -$DIRNAME/kube/deploy.sh echo "Airflow environment on kubernetes is good to go!" diff --git a/scripts/ci/run-ci.sh b/scripts/ci/run-ci.sh index f2815bbd95979..1a65bf1ef6443 100755 --- a/scripts/ci/run-ci.sh +++ b/scripts/ci/run-ci.sh @@ -24,7 +24,7 @@ DIRNAME=$(cd "$(dirname "$0")"; pwd) AIRFLOW_ROOT="$DIRNAME/../.." # Fix file permissions -sudo chown -R airflow.airflow . $HOME/.wheelhouse/ $HOME/.cache/pip +sudo chown -R airflow.airflow . $HOME/.cache $HOME/.wheelhouse/ $HOME/.cache/pip $HOME/.kube $HOME/.minikube if [[ $PYTHON_VERSION == '3' ]]; then PIP=pip3 @@ -41,7 +41,13 @@ if [ -z "$KUBERNETES_VERSION" ]; then tox -e $TOX_ENV else - KUBERNETES_VERSION=${KUBERNETES_VERSION} $DIRNAME/kubernetes/setup_kubernetes.sh && \ + # This script runs inside a container, the path of the kubernetes certificate + # is /home/travis/.minikube/client.crt but the user in the container is `airflow` + if [ ! -d /home/travis ]; then + sudo mkdir -p /home/travis + fi + sudo ln -s /home/airflow/.minikube /home/travis/.minikube + tox -e $TOX_ENV -- tests.contrib.minikube \ --with-coverage \ --cover-erase \ diff --git a/tests/contrib/minikube/test_kubernetes_executor.py b/tests/contrib/minikube/test_kubernetes_executor.py index 45d4124d07973..23bdfaec8d5bf 100644 --- a/tests/contrib/minikube/test_kubernetes_executor.py +++ b/tests/contrib/minikube/test_kubernetes_executor.py @@ -16,6 +16,7 @@ # under the License. +import os import unittest from subprocess import check_call, check_output import requests.exceptions @@ -25,18 +26,25 @@ import re try: - check_call(["kubectl", "get", "pods"]) + check_call(["/usr/local/bin/kubectl", "get", "pods"]) except Exception as e: - raise unittest.SkipTest( - "Kubernetes integration tests require a minikube cluster;" - "Skipping tests {}".format(e) - ) + if os.environ.get('KUBERNETES_VERSION'): + raise e + else: + raise unittest.SkipTest( + "Kubernetes integration tests require a minikube cluster;" + "Skipping tests {}".format(e) + ) def get_minikube_host(): - host_ip = check_output(['minikube', 'ip']) - if six.PY3: - host_ip = host_ip.decode('UTF-8') + if "MINIKUBE_IP" in os.environ: + host_ip = os.environ['MINIKUBE_IP'] + else: + host_ip = check_output(['/usr/local/bin/minikube', 'ip']) + if six.PY3: + host_ip = host_ip.decode('UTF-8') + host = '{}:30809'.format(host_ip.strip()) return host diff --git a/tests/contrib/minikube/test_kubernetes_pod_operator.py b/tests/contrib/minikube/test_kubernetes_pod_operator.py index 595d7aa8b43e5..f39fcdb03d56a 100644 --- a/tests/contrib/minikube/test_kubernetes_pod_operator.py +++ b/tests/contrib/minikube/test_kubernetes_pod_operator.py @@ -29,12 +29,15 @@ from airflow.contrib.kubernetes.volume import Volume try: - check_call(["kubectl", "get", "pods"]) + check_call(["/usr/local/bin/kubectl", "get", "pods"]) except Exception as e: - raise unittest.SkipTest( - "Kubernetes integration tests require a minikube cluster;" - "Skipping tests {}".format(e) - ) + if os.environ.get('KUBERNETES_VERSION'): + raise e + else: + raise unittest.SkipTest( + "Kubernetes integration tests require a minikube cluster;" + "Skipping tests {}".format(e) + ) class KubernetesPodOperatorTest(unittest.TestCase): diff --git a/tests/core.py b/tests/core.py index b937178a9ef8c..a135857e375ad 100644 --- a/tests/core.py +++ b/tests/core.py @@ -97,7 +97,8 @@ def reset(dag_id=TEST_DAG_ID): configuration.conf.load_test_config() -reset() +if os.environ.get('KUBERNETES_VERSION') is None: + reset() class OperatorSubclass(BaseOperator): diff --git a/tox.ini b/tox.ini index c4b74a1e55345..1ba7d84fea226 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,7 @@ # under the License. [tox] -envlist = flake8,{py27,py35}-backend_{mysql,sqlite,postgres} +envlist = flake8,{py27,py35}-backend_{mysql,sqlite,postgres}-env_{docker,kubernetes} skipsdist = True [global] @@ -57,10 +57,10 @@ passenv = * commands = pip wheel --progress-bar off -w {homedir}/.wheelhouse -f {homedir}/.wheelhouse -e .[devel_ci] pip install --progress-bar off --find-links={homedir}/.wheelhouse --no-index -e .[devel_ci] - {toxinidir}/scripts/ci/1-setup-env.sh - {toxinidir}/scripts/ci/2-setup-kdc.sh - {toxinidir}/scripts/ci/3-setup-databases.sh - {toxinidir}/scripts/ci/4-load-data.sh + env_docker: {toxinidir}/scripts/ci/1-setup-env.sh + env_docker: {toxinidir}/scripts/ci/2-setup-kdc.sh + env_docker: {toxinidir}/scripts/ci/3-setup-databases.sh + env_docker: {toxinidir}/scripts/ci/4-load-data.sh {toxinidir}/scripts/ci/5-run-tests.sh [] {toxinidir}/scripts/ci/6-check-license.sh codecov -e TOXENV