From 0718706a8d9a816e832c4f6f945533319cfad88d Mon Sep 17 00:00:00 2001 From: Jonathan Camile Date: Thu, 29 Mar 2018 16:53:19 +0200 Subject: [PATCH 1/3] Add config to have customizable pip and airflow executables --- defaults/main.yml | 3 +++ tasks/config.yml | 2 +- tasks/install.yml | 5 +++++ templates/airflow-flower.service.j2 | 2 +- templates/airflow-scheduler.service.j2 | 2 +- templates/airflow-webserver.service.j2 | 2 +- templates/airflow-worker.service.j2 | 2 +- 7 files changed, 13 insertions(+), 5 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index 05ff80d..1247356 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -4,6 +4,9 @@ airflow_version: 1.8.1 airflow_extra_packages: +airflow_pip_executable: "pip" +airflow_executable: "/usr/local/bin/airflow" + airflow_required_libs: - python-pip - acl diff --git a/tasks/config.yml b/tasks/config.yml index 781c9f8..36b5557 100644 --- a/tasks/config.yml +++ b/tasks/config.yml @@ -35,7 +35,7 @@ - restart airflow-flower - name: Airflow | Initializing DB - command: /usr/local/bin/airflow initdb + command: "{{ airflow_executable }} initdb" environment: AIRFLOW_HOME: "{{ airflow_home }}" become: true diff --git a/tasks/install.yml b/tasks/install.yml index 2225393..5070d46 100644 --- a/tasks/install.yml +++ b/tasks/install.yml @@ -33,12 +33,14 @@ - name: Airflow | Installing proper Celery version pip: + executable: "{{ airflow_pip_executable }}" name: celery version: "{{ celery_version }}" when: airflow_executor == "CeleryExecutor" - name: Airflow | Installing extra Celery packages pip: + executable: "{{ airflow_pip_executable }}" name: celery[{{ item }}] version: "{{ celery_version }}" with_items: "{{ celery_extra_packages }}" @@ -51,6 +53,7 @@ - name: Airflow | Installing Airflow pip: + executable: "{{ airflow_pip_executable }}" name: apache-airflow version: "{{ airflow_version }}" extra_args: --no-cache-dir @@ -58,6 +61,7 @@ - name: Airflow | Installing Airflow Extra Packages pip: + executable: "{{ airflow_pip_executable }}" name: apache-airflow[{{ item }}] version: "{{ airflow_version }}" with_items: "{{ airflow_extra_packages }}" @@ -65,6 +69,7 @@ - name: Airflow | Installing DAGs dependencies pip: + executable: "{{ airflow_pip_executable }}" name: "{{ item.name }}" version: "{{ item.version }}" with_items: "{{ dags_dependencies }}" diff --git a/templates/airflow-flower.service.j2 b/templates/airflow-flower.service.j2 index 4caa109..122612d 100644 --- a/templates/airflow-flower.service.j2 +++ b/templates/airflow-flower.service.j2 @@ -21,7 +21,7 @@ EnvironmentFile={{ airflow_environment_file_folder }}/airflow User={{ airflow_user }} Group={{ airflow_group }} Type=simple -ExecStart=/usr/local/bin/airflow flower --pid /run/airflow/flower.pid +ExecStart={{ airflow_executable }} flower --pid /run/airflow/flower.pid Restart=on-failure RestartSec=10s RuntimeDirectory=airflow diff --git a/templates/airflow-scheduler.service.j2 b/templates/airflow-scheduler.service.j2 index 637872f..54a26a2 100644 --- a/templates/airflow-scheduler.service.j2 +++ b/templates/airflow-scheduler.service.j2 @@ -21,7 +21,7 @@ EnvironmentFile={{ airflow_environment_file_folder }}/airflow User={{ airflow_user }} Group={{ airflow_group }} Type=simple -ExecStart=/usr/local/bin/airflow scheduler -n ${SCHEDULER_RUNS} --pid /run/airflow/scheduler.pid +ExecStart={{ airflow_executable }} scheduler -n ${SCHEDULER_RUNS} --pid /run/airflow/scheduler.pid Restart=always RestartSec=5s RuntimeDirectory=airflow diff --git a/templates/airflow-webserver.service.j2 b/templates/airflow-webserver.service.j2 index 6192235..22393ef 100644 --- a/templates/airflow-webserver.service.j2 +++ b/templates/airflow-webserver.service.j2 @@ -21,7 +21,7 @@ EnvironmentFile={{ airflow_environment_file_folder }}/airflow User={{ airflow_user }} Group={{ airflow_group }} Type=simple -ExecStart=/usr/local/bin/airflow webserver --pid /run/airflow/webserver.pid +ExecStart={{ airflow_executable }} webserver --pid /run/airflow/webserver.pid Restart=on-failure RestartSec=5s PrivateTmp=true diff --git a/templates/airflow-worker.service.j2 b/templates/airflow-worker.service.j2 index b22899d..122e90a 100644 --- a/templates/airflow-worker.service.j2 +++ b/templates/airflow-worker.service.j2 @@ -21,7 +21,7 @@ EnvironmentFile={{ airflow_environment_file_folder }}/airflow User={{ airflow_user }} Group={{ airflow_group }} Type=simple -ExecStart=/usr/local/bin/airflow worker --pid /run/airflow/worker.pid +ExecStart={{ airflow_executable }} worker --pid /run/airflow/worker.pid Restart=on-failure RestartSec=10s RuntimeDirectory=airflow From 29e1ab2121bb485feddb948223d0ecb330dcb2cd Mon Sep 17 00:00:00 2001 From: Jonathan Camile Date: Fri, 30 Mar 2018 11:35:40 +0200 Subject: [PATCH 2/3] Edit environnement template file to allow additional variables --- defaults/main.yml | 1 + templates/airflow-environment-file.j2 | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/defaults/main.yml b/defaults/main.yml index 1247356..abde42e 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -39,6 +39,7 @@ airflow_dags_folder: "{{ airflow_home }}/dags" airflow_logs_folder: /var/log/airflow airflow_child_process_log_folder: "{{ airflow_logs_folder }}/scheduler" airflow_environment_file_folder: /etc/sysconfig +airflow_environment_extra_vars: [] # Allowing playbooks to provide external config files&templates airflow_extra_conf_path: "{{ playbook_dir }}/files/airflow" diff --git a/templates/airflow-environment-file.j2 b/templates/airflow-environment-file.j2 index 74aca45..c375d97 100644 --- a/templates/airflow-environment-file.j2 +++ b/templates/airflow-environment-file.j2 @@ -21,4 +21,8 @@ # required setting, 0 sets it to unlimited. Scheduler will get restart after every X runs SCHEDULER_RUNS={{ airflow_scheduler_runs }} -AIRFLOW_HOME={{ airflow_home }} \ No newline at end of file +AIRFLOW_HOME={{ airflow_home }} + +{% for extra_var in airflow_environment_extra_vars %} +{{ extra_var.name }}={{ extra_var.value }} +{% endfor %} From 4814ba371f1d749b6dbd5e4b2ec6b208125007b3 Mon Sep 17 00:00:00 2001 From: Jonathan Camile Date: Tue, 3 Apr 2018 09:45:33 +0200 Subject: [PATCH 3/3] Add example for `airflow_environment_extra_vars` configuration --- defaults/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/defaults/main.yml b/defaults/main.yml index abde42e..f17e01d 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -40,6 +40,8 @@ airflow_logs_folder: /var/log/airflow airflow_child_process_log_folder: "{{ airflow_logs_folder }}/scheduler" airflow_environment_file_folder: /etc/sysconfig airflow_environment_extra_vars: [] + # - name: PATH + # value: "/custom/path/bin:$PATH" # Allowing playbooks to provide external config files&templates airflow_extra_conf_path: "{{ playbook_dir }}/files/airflow"