From 35849add5c967ab5a4fe49d2b75404e9facd956e Mon Sep 17 00:00:00 2001 From: Cameron Yick Date: Mon, 15 Oct 2018 21:16:33 -0400 Subject: [PATCH 1/9] Set up Travis configuration file --- .travis.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..8a319a7 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,12 @@ +env: + - DOCKER_COMPOSE_VERSION=1.9.0 + +before_install: + - sudo rm /usr/local/bin/docker-compose + - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose + - chmod +x docker-compose + - sudo mv docker-compose /usr/local/bin + +script: + - cd tests/airflow1.8-py2 && docker-compose up -d + - curl http://localhost:8080/admin/metrics From 8b91a5233fe41143e5056091f0d930810e4cd465 Mon Sep 17 00:00:00 2001 From: Cameron Yick Date: Mon, 15 Oct 2018 21:36:54 -0400 Subject: [PATCH 2/9] Wait for airflow to come up before pinging endpoint --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 8a319a7..280c220 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,4 +9,5 @@ before_install: script: - cd tests/airflow1.8-py2 && docker-compose up -d + - pwd && sleep 30 # not sure how to dynamically wait - curl http://localhost:8080/admin/metrics From 2c2afc173d3bd548170e7053a1d1810a32042881 Mon Sep 17 00:00:00 2001 From: Cameron Yick Date: Mon, 15 Oct 2018 21:40:41 -0400 Subject: [PATCH 3/9] Add Python3 Test and Docker Service --- .travis.yml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 280c220..c8ede81 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,13 @@ +sudo: required + +services: + - docker + +# Docker-compose setup +# https://docs.travis-ci.com/user/docker/ env: - DOCKER_COMPOSE_VERSION=1.9.0 + - AIRFLOW_SLEEP_DURATION=60 # Number of seconds to wait for airflow to start before_install: - sudo rm /usr/local/bin/docker-compose @@ -8,6 +16,15 @@ before_install: - sudo mv docker-compose /usr/local/bin script: + + # Test Airflow 1.8 with Python 2 - cd tests/airflow1.8-py2 && docker-compose up -d - - pwd && sleep 30 # not sure how to dynamically wait - - curl http://localhost:8080/admin/metrics + - pwd && sleep ${AIRFLOW_SLEEP_DURATION} + - curl http://localhost:8080/admin/metrics/ + - docker-compose down + + # Test Airflow 1.9 with Python 3 + - cd ../../tests/airflow1.9-py3 && docker-compose up -d + - pwd && sleep ${AIRFLOW_SLEEP_DURATION} + - curl http://localhost:8080/admin/metrics/ + - docker-compose down From cf65876948a1db91007ca4dc1f81f5feada73c3e Mon Sep 17 00:00:00 2001 From: Cameron Yick Date: Mon, 15 Oct 2018 21:55:39 -0400 Subject: [PATCH 4/9] Make Environment Variables Global --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c8ede81..e11edce 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,8 +6,9 @@ services: # Docker-compose setup # https://docs.travis-ci.com/user/docker/ env: - - DOCKER_COMPOSE_VERSION=1.9.0 - - AIRFLOW_SLEEP_DURATION=60 # Number of seconds to wait for airflow to start + global: + - DOCKER_COMPOSE_VERSION=1.9.0 + - AIRFLOW_SLEEP_DURATION=60 # Number of seconds to wait for airflow to start before_install: - sudo rm /usr/local/bin/docker-compose From 721575d050088272b373e76b9a2e868ce0ed9fe2 Mon Sep 17 00:00:00 2001 From: Cameron Yick Date: Mon, 15 Oct 2018 22:01:44 -0400 Subject: [PATCH 5/9] Fail with nonzero status if status code is not 200 --- .travis.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index e11edce..7587b6e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,8 @@ env: global: - DOCKER_COMPOSE_VERSION=1.9.0 - AIRFLOW_SLEEP_DURATION=60 # Number of seconds to wait for airflow to start + - METRICS_ENDPOINT=http://localhost:8080/admin/metrics/ + - CURL_FLAGS=--show-error --fail before_install: - sudo rm /usr/local/bin/docker-compose @@ -20,12 +22,12 @@ script: # Test Airflow 1.8 with Python 2 - cd tests/airflow1.8-py2 && docker-compose up -d - - pwd && sleep ${AIRFLOW_SLEEP_DURATION} - - curl http://localhost:8080/admin/metrics/ + - sleep ${AIRFLOW_SLEEP_DURATION} + - curl ${CURL_FLAGS} ${METRICS_ENDPOINT} - docker-compose down # Test Airflow 1.9 with Python 3 - cd ../../tests/airflow1.9-py3 && docker-compose up -d - - pwd && sleep ${AIRFLOW_SLEEP_DURATION} - - curl http://localhost:8080/admin/metrics/ + - sleep ${AIRFLOW_SLEEP_DURATION} + - curl ${CURL_FLAGS} ${METRICS_ENDPOINT} - docker-compose down From db7eb395a3ba0b48ebc63f1b6fa37c449444146d Mon Sep 17 00:00:00 2001 From: Cameron Yick Date: Mon, 15 Oct 2018 22:08:10 -0400 Subject: [PATCH 6/9] Factor integration test into dedicated bash script --- .travis.yml | 14 +++++--------- tests/test_metrics_up.sh | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 9 deletions(-) create mode 100644 tests/test_metrics_up.sh diff --git a/.travis.yml b/.travis.yml index 7587b6e..89630ac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,26 +8,22 @@ services: env: global: - DOCKER_COMPOSE_VERSION=1.9.0 - - AIRFLOW_SLEEP_DURATION=60 # Number of seconds to wait for airflow to start - - METRICS_ENDPOINT=http://localhost:8080/admin/metrics/ - - CURL_FLAGS=--show-error --fail - + - INTEGRATION_TEST_SCRIPT=test_metrics_up.sh + before_install: - sudo rm /usr/local/bin/docker-compose - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose - chmod +x docker-compose - sudo mv docker-compose /usr/local/bin + - chmod +x tests/test_metrics_up.sh script: - # Test Airflow 1.8 with Python 2 - cd tests/airflow1.8-py2 && docker-compose up -d - - sleep ${AIRFLOW_SLEEP_DURATION} - - curl ${CURL_FLAGS} ${METRICS_ENDPOINT} + - ../${INTEGRATION_TEST_SCRIPT} - docker-compose down # Test Airflow 1.9 with Python 3 - cd ../../tests/airflow1.9-py3 && docker-compose up -d - - sleep ${AIRFLOW_SLEEP_DURATION} - - curl ${CURL_FLAGS} ${METRICS_ENDPOINT} + - ../${INTEGRATION_TEST_SCRIPT} - docker-compose down diff --git a/tests/test_metrics_up.sh b/tests/test_metrics_up.sh new file mode 100644 index 0000000..109987f --- /dev/null +++ b/tests/test_metrics_up.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Integration test that +# 1. Waits for Airflow to come up within 60 seconds +# 2. Ensures the Metrics endpoint returned a valid response +# Depends on Curl + +AIRFLOW_SLEEP_DURATION=70 # Number of seconds to wait for airflow to start +METRICS_ENDPOINT="http://localhost:8080/admin/metrics/" + +# Return nonzero status code if endpoint does not return 200 +CURL_FLAGS="--show-error --fail" + +sleep ${AIRFLOW_SLEEP_DURATION} +curl ${CURL_FLAGS} ${METRICS_ENDPOINT} + +# TODO: validate the contents of the CURLed data From f6bdaa787cfa2d2a4d7661d88f66314ea45dc30f Mon Sep 17 00:00:00 2001 From: Cameron Yick Date: Tue, 16 Oct 2018 01:34:55 -0400 Subject: [PATCH 7/9] Relax Permissions for Python3 Run as root user due to changes in python logging --- .travis.yml | 9 +++++---- tests/airflow1.8-py2/docker-compose.yml | 1 - tests/airflow1.9-py3/airflow/build/Dockerfile | 5 ++++- tests/test_metrics_up.sh | 5 ++++- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 89630ac..5fc7e83 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -sudo: required +sudo: false services: - docker @@ -16,14 +16,15 @@ before_install: - chmod +x docker-compose - sudo mv docker-compose /usr/local/bin - chmod +x tests/test_metrics_up.sh + script: # Test Airflow 1.8 with Python 2 - cd tests/airflow1.8-py2 && docker-compose up -d - - ../${INTEGRATION_TEST_SCRIPT} + - ${TRAVIS_BUILD_DIR}/tests/${INTEGRATION_TEST_SCRIPT} - docker-compose down # Test Airflow 1.9 with Python 3 - - cd ../../tests/airflow1.9-py3 && docker-compose up -d - - ../${INTEGRATION_TEST_SCRIPT} + - cd ${TRAVIS_BUILD_DIR}/tests/airflow1.9-py3 && docker-compose up -d + - ${TRAVIS_BUILD_DIR}/tests/${INTEGRATION_TEST_SCRIPT} - docker-compose down diff --git a/tests/airflow1.8-py2/docker-compose.yml b/tests/airflow1.8-py2/docker-compose.yml index d8ae9ac..e99eaaa 100644 --- a/tests/airflow1.8-py2/docker-compose.yml +++ b/tests/airflow1.8-py2/docker-compose.yml @@ -1,6 +1,5 @@ version: '2' - services: exporter_postgresql: diff --git a/tests/airflow1.9-py3/airflow/build/Dockerfile b/tests/airflow1.9-py3/airflow/build/Dockerfile index 0c0f864..e5695a9 100644 --- a/tests/airflow1.9-py3/airflow/build/Dockerfile +++ b/tests/airflow1.9-py3/airflow/build/Dockerfile @@ -4,4 +4,7 @@ USER root RUN pip3 install prometheus_client -USER airflow +# RUN mkdir -p /usr/local/airflow/logs && chown airflow -R /usr/local/airflow/logs +# # RUN chown -R airflow /usr/local/airflow + +# USER airflow diff --git a/tests/test_metrics_up.sh b/tests/test_metrics_up.sh index 109987f..34be915 100644 --- a/tests/test_metrics_up.sh +++ b/tests/test_metrics_up.sh @@ -4,13 +4,16 @@ # 2. Ensures the Metrics endpoint returned a valid response # Depends on Curl -AIRFLOW_SLEEP_DURATION=70 # Number of seconds to wait for airflow to start +AIRFLOW_SLEEP_DURATION=90 # Number of seconds to wait for airflow to start +ADMIN_ENDPOINT="http://localhost:8080/admin/" METRICS_ENDPOINT="http://localhost:8080/admin/metrics/" # Return nonzero status code if endpoint does not return 200 CURL_FLAGS="--show-error --fail" +echo "Waiting ${AIRFLOW_SLEEP_DURATION} seconds for Airflow to start before pinging" sleep ${AIRFLOW_SLEEP_DURATION} +curl ${CURL_FLAGS} --silent ${ADMIN_ENDPOINT} && echo 'Admin console is up' curl ${CURL_FLAGS} ${METRICS_ENDPOINT} # TODO: validate the contents of the CURLed data From 8548a3d58f510a07944077b10b6cba4f321d5974 Mon Sep 17 00:00:00 2001 From: Cameron Yick Date: Wed, 17 Oct 2018 20:58:25 -0400 Subject: [PATCH 8/9] Run airflow using host user permissions instead of root WIP: disable python2 pipeline --- tests/airflow1.9-py3/airflow/build/Dockerfile | 6 ++---- tests/airflow1.9-py3/docker-compose.yml | 1 + tests/test_metrics_up.sh | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/airflow1.9-py3/airflow/build/Dockerfile b/tests/airflow1.9-py3/airflow/build/Dockerfile index e5695a9..8341b50 100644 --- a/tests/airflow1.9-py3/airflow/build/Dockerfile +++ b/tests/airflow1.9-py3/airflow/build/Dockerfile @@ -4,7 +4,5 @@ USER root RUN pip3 install prometheus_client -# RUN mkdir -p /usr/local/airflow/logs && chown airflow -R /usr/local/airflow/logs -# # RUN chown -R airflow /usr/local/airflow - -# USER airflow +# Needed to write to mounted directory with proper permissions +USER ${HOST_USER_ID} diff --git a/tests/airflow1.9-py3/docker-compose.yml b/tests/airflow1.9-py3/docker-compose.yml index b655214..8ccf3b6 100644 --- a/tests/airflow1.9-py3/docker-compose.yml +++ b/tests/airflow1.9-py3/docker-compose.yml @@ -24,6 +24,7 @@ services: - POSTGRES_USER=airflow - POSTGRES_PASSWORD=airflowpass - POSTGRES_DB=airflow + - HOST_USER_ID=$UID # need to write to mounted files volumes: - ../dags:/usr/local/airflow/dags - ../..:/usr/local/airflow/plugins/airflow-exporter-0.000 diff --git a/tests/test_metrics_up.sh b/tests/test_metrics_up.sh index 34be915..4c88b95 100644 --- a/tests/test_metrics_up.sh +++ b/tests/test_metrics_up.sh @@ -13,7 +13,7 @@ CURL_FLAGS="--show-error --fail" echo "Waiting ${AIRFLOW_SLEEP_DURATION} seconds for Airflow to start before pinging" sleep ${AIRFLOW_SLEEP_DURATION} -curl ${CURL_FLAGS} --silent ${ADMIN_ENDPOINT} && echo 'Admin console is up' +curl ${CURL_FLAGS} --silent --output /dev/null ${ADMIN_ENDPOINT} && echo 'Admin console is up' curl ${CURL_FLAGS} ${METRICS_ENDPOINT} # TODO: validate the contents of the CURLed data From c601215d0d27027bc053de51d13bcf05985e0dfc Mon Sep 17 00:00:00 2001 From: cleverCat Date: Wed, 14 Nov 2018 12:13:10 +0300 Subject: [PATCH 9/9] fix run test in travis --- .travis.yml | 2 ++ CHANGELOG.md | 5 +++++ tests/airflow1.9-py3/airflow/build/Dockerfile | 3 +-- tests/airflow1.9-py3/docker-compose.yml | 1 - 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5fc7e83..58b2e77 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,6 +25,8 @@ script: - docker-compose down # Test Airflow 1.9 with Python 3 + - mkdir ${TRAVIS_BUILD_DIR}/tests/airflow1.9-py3/airflow/logs + - chmod -R 777 ${TRAVIS_BUILD_DIR}/tests/airflow1.9-py3/airflow/logs - cd ${TRAVIS_BUILD_DIR}/tests/airflow1.9-py3 && docker-compose up -d - ${TRAVIS_BUILD_DIR}/tests/${INTEGRATION_TEST_SCRIPT} - docker-compose down diff --git a/CHANGELOG.md b/CHANGELOG.md index f0285c1..4b0d0cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ # Changelog All notable changes to this project will be documented in this file. +## 0.4.2 - 2018-11-13 + +- [#13](https://github.com/epoch8/airflow-exporter/pull/20): Added test script and travis file by @hydrosquall +- [#13](https://github.com/epoch8/airflow-exporter/pull/27): fix run test in travis @cleverCat + ## 0.4 - 2018-10-15 - Fix [#14](https://github.com/epoch8/airflow-exporter/issues/14): Airflow 1.10 compatibility by @jmcarp diff --git a/tests/airflow1.9-py3/airflow/build/Dockerfile b/tests/airflow1.9-py3/airflow/build/Dockerfile index 8341b50..0c0f864 100644 --- a/tests/airflow1.9-py3/airflow/build/Dockerfile +++ b/tests/airflow1.9-py3/airflow/build/Dockerfile @@ -4,5 +4,4 @@ USER root RUN pip3 install prometheus_client -# Needed to write to mounted directory with proper permissions -USER ${HOST_USER_ID} +USER airflow diff --git a/tests/airflow1.9-py3/docker-compose.yml b/tests/airflow1.9-py3/docker-compose.yml index 8ccf3b6..b655214 100644 --- a/tests/airflow1.9-py3/docker-compose.yml +++ b/tests/airflow1.9-py3/docker-compose.yml @@ -24,7 +24,6 @@ services: - POSTGRES_USER=airflow - POSTGRES_PASSWORD=airflowpass - POSTGRES_DB=airflow - - HOST_USER_ID=$UID # need to write to mounted files volumes: - ../dags:/usr/local/airflow/dags - ../..:/usr/local/airflow/plugins/airflow-exporter-0.000