diff --git a/defaults/main.yml b/defaults/main.yml index 05ff80d..f17e01d 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -4,6 +4,9 @@ airflow_version: 1.8.1 airflow_extra_packages: +airflow_pip_executable: "pip" +airflow_executable: "/usr/local/bin/airflow" + airflow_required_libs: - python-pip - acl @@ -36,6 +39,9 @@ airflow_dags_folder: "{{ airflow_home }}/dags" airflow_logs_folder: /var/log/airflow airflow_child_process_log_folder: "{{ airflow_logs_folder }}/scheduler" airflow_environment_file_folder: /etc/sysconfig +airflow_environment_extra_vars: [] + # - name: PATH + # value: "/custom/path/bin:$PATH" # Allowing playbooks to provide external config files&templates airflow_extra_conf_path: "{{ playbook_dir }}/files/airflow" diff --git a/tasks/config.yml b/tasks/config.yml index 781c9f8..36b5557 100644 --- a/tasks/config.yml +++ b/tasks/config.yml @@ -35,7 +35,7 @@ - restart airflow-flower - name: Airflow | Initializing DB - command: /usr/local/bin/airflow initdb + command: "{{ airflow_executable }} initdb" environment: AIRFLOW_HOME: "{{ airflow_home }}" become: true diff --git a/tasks/install.yml b/tasks/install.yml index 2225393..5070d46 100644 --- a/tasks/install.yml +++ b/tasks/install.yml @@ -33,12 +33,14 @@ - name: Airflow | Installing proper Celery version pip: + executable: "{{ airflow_pip_executable }}" name: celery version: "{{ celery_version }}" when: airflow_executor == "CeleryExecutor" - name: Airflow | Installing extra Celery packages pip: + executable: "{{ airflow_pip_executable }}" name: celery[{{ item }}] version: "{{ celery_version }}" with_items: "{{ celery_extra_packages }}" @@ -51,6 +53,7 @@ - name: Airflow | Installing Airflow pip: + executable: "{{ airflow_pip_executable }}" name: apache-airflow version: "{{ airflow_version }}" extra_args: --no-cache-dir @@ -58,6 +61,7 @@ - name: Airflow | Installing Airflow Extra Packages pip: + executable: "{{ airflow_pip_executable }}" name: apache-airflow[{{ item }}] version: "{{ airflow_version }}" with_items: "{{ airflow_extra_packages }}" @@ -65,6 +69,7 @@ - name: Airflow | Installing DAGs dependencies pip: + executable: "{{ airflow_pip_executable }}" name: "{{ item.name }}" version: "{{ item.version }}" with_items: "{{ dags_dependencies }}" diff --git a/templates/airflow-environment-file.j2 b/templates/airflow-environment-file.j2 index 74aca45..c375d97 100644 --- a/templates/airflow-environment-file.j2 +++ b/templates/airflow-environment-file.j2 @@ -21,4 +21,8 @@ # required setting, 0 sets it to unlimited. Scheduler will get restart after every X runs SCHEDULER_RUNS={{ airflow_scheduler_runs }} -AIRFLOW_HOME={{ airflow_home }} \ No newline at end of file +AIRFLOW_HOME={{ airflow_home }} + +{% for extra_var in airflow_environment_extra_vars %} +{{ extra_var.name }}={{ extra_var.value }} +{% endfor %} diff --git a/templates/airflow-flower.service.j2 b/templates/airflow-flower.service.j2 index 4caa109..122612d 100644 --- a/templates/airflow-flower.service.j2 +++ b/templates/airflow-flower.service.j2 @@ -21,7 +21,7 @@ EnvironmentFile={{ airflow_environment_file_folder }}/airflow User={{ airflow_user }} Group={{ airflow_group }} Type=simple -ExecStart=/usr/local/bin/airflow flower --pid /run/airflow/flower.pid +ExecStart={{ airflow_executable }} flower --pid /run/airflow/flower.pid Restart=on-failure RestartSec=10s RuntimeDirectory=airflow diff --git a/templates/airflow-scheduler.service.j2 b/templates/airflow-scheduler.service.j2 index 637872f..54a26a2 100644 --- a/templates/airflow-scheduler.service.j2 +++ b/templates/airflow-scheduler.service.j2 @@ -21,7 +21,7 @@ EnvironmentFile={{ airflow_environment_file_folder }}/airflow User={{ airflow_user }} Group={{ airflow_group }} Type=simple -ExecStart=/usr/local/bin/airflow scheduler -n ${SCHEDULER_RUNS} --pid /run/airflow/scheduler.pid +ExecStart={{ airflow_executable }} scheduler -n ${SCHEDULER_RUNS} --pid /run/airflow/scheduler.pid Restart=always RestartSec=5s RuntimeDirectory=airflow diff --git a/templates/airflow-webserver.service.j2 b/templates/airflow-webserver.service.j2 index 6192235..22393ef 100644 --- a/templates/airflow-webserver.service.j2 +++ b/templates/airflow-webserver.service.j2 @@ -21,7 +21,7 @@ EnvironmentFile={{ airflow_environment_file_folder }}/airflow User={{ airflow_user }} Group={{ airflow_group }} Type=simple -ExecStart=/usr/local/bin/airflow webserver --pid /run/airflow/webserver.pid +ExecStart={{ airflow_executable }} webserver --pid /run/airflow/webserver.pid Restart=on-failure RestartSec=5s PrivateTmp=true diff --git a/templates/airflow-worker.service.j2 b/templates/airflow-worker.service.j2 index b22899d..122e90a 100644 --- a/templates/airflow-worker.service.j2 +++ b/templates/airflow-worker.service.j2 @@ -21,7 +21,7 @@ EnvironmentFile={{ airflow_environment_file_folder }}/airflow User={{ airflow_user }} Group={{ airflow_group }} Type=simple -ExecStart=/usr/local/bin/airflow worker --pid /run/airflow/worker.pid +ExecStart={{ airflow_executable }} worker --pid /run/airflow/worker.pid Restart=on-failure RestartSec=10s RuntimeDirectory=airflow