Merge pull request #34 from MeilleursAgents/features/use_in_venv

Add config to have customizable pip and airflow executables
idealista · Apr 3, 2018 · 47aeab9 · 47aeab9
2 parents 27511f5 + 4814ba3
commit 47aeab9
Show file tree

Hide file tree

Showing 8 changed files with 21 additions and 6 deletions.
diff --git a/defaults/main.yml b/defaults/main.yml
@@ -4,6 +4,9 @@
 airflow_version: 1.8.1
 airflow_extra_packages:
 
+airflow_pip_executable: "pip"
+airflow_executable: "/usr/local/bin/airflow"
+
 airflow_required_libs:
   - python-pip
   - acl
@@ -36,6 +39,9 @@ airflow_dags_folder: "{{ airflow_home }}/dags"
 airflow_logs_folder: /var/log/airflow
 airflow_child_process_log_folder: "{{ airflow_logs_folder }}/scheduler"
 airflow_environment_file_folder: /etc/sysconfig
+airflow_environment_extra_vars: []
+  # - name: PATH
+  #   value: "/custom/path/bin:$PATH"
 
 # Allowing playbooks to provide external config files&templates
 airflow_extra_conf_path: "{{ playbook_dir }}/files/airflow"

diff --git a/tasks/config.yml b/tasks/config.yml
@@ -35,7 +35,7 @@
     - restart airflow-flower
 
 - name: Airflow | Initializing DB
-  command: /usr/local/bin/airflow initdb
+  command: "{{ airflow_executable }} initdb"
   environment:
     AIRFLOW_HOME: "{{ airflow_home }}"
   become: true

diff --git a/tasks/install.yml b/tasks/install.yml
@@ -33,12 +33,14 @@
 
 - name: Airflow | Installing proper Celery version
   pip:
+    executable: "{{ airflow_pip_executable }}"
     name: celery
     version: "{{ celery_version }}"
   when: airflow_executor == "CeleryExecutor"
 
 - name: Airflow | Installing extra Celery packages
   pip:
+    executable: "{{ airflow_pip_executable }}"
     name: celery[{{ item }}]
     version: "{{ celery_version }}"
   with_items: "{{ celery_extra_packages }}"
@@ -51,20 +53,23 @@
 
 - name: Airflow | Installing Airflow
   pip:
+    executable: "{{ airflow_pip_executable }}"
     name: apache-airflow
     version: "{{ airflow_version }}"
     extra_args: --no-cache-dir
   register: airflow_install
 
 - name: Airflow | Installing Airflow Extra Packages
   pip:
+    executable: "{{ airflow_pip_executable }}"
     name: apache-airflow[{{ item }}]
     version: "{{ airflow_version }}"
   with_items: "{{ airflow_extra_packages }}"
   when: airflow_extra_packages
 
 - name: Airflow | Installing DAGs dependencies
   pip:
+    executable: "{{ airflow_pip_executable }}"
     name: "{{ item.name }}"
     version: "{{ item.version }}"
   with_items: "{{ dags_dependencies }}"

diff --git a/templates/airflow-environment-file.j2 b/templates/airflow-environment-file.j2
@@ -21,4 +21,8 @@
 # required setting, 0 sets it to unlimited. Scheduler will get restart after every X runs
 SCHEDULER_RUNS={{ airflow_scheduler_runs }}
 
-AIRFLOW_HOME={{ airflow_home }}
+AIRFLOW_HOME={{ airflow_home }}
+
+{% for extra_var in airflow_environment_extra_vars %}
+{{ extra_var.name }}={{ extra_var.value }}
+{% endfor %}
diff --git a/templates/airflow-flower.service.j2 b/templates/airflow-flower.service.j2
@@ -21,7 +21,7 @@ EnvironmentFile={{ airflow_environment_file_folder }}/airflow
 User={{ airflow_user }}
 Group={{ airflow_group }}
 Type=simple
-ExecStart=/usr/local/bin/airflow flower --pid /run/airflow/flower.pid
+ExecStart={{ airflow_executable }} flower --pid /run/airflow/flower.pid
 Restart=on-failure
 RestartSec=10s
 RuntimeDirectory=airflow

diff --git a/templates/airflow-scheduler.service.j2 b/templates/airflow-scheduler.service.j2
@@ -21,7 +21,7 @@ EnvironmentFile={{ airflow_environment_file_folder }}/airflow
 User={{ airflow_user }}
 Group={{ airflow_group }}
 Type=simple
-ExecStart=/usr/local/bin/airflow scheduler -n ${SCHEDULER_RUNS} --pid /run/airflow/scheduler.pid
+ExecStart={{ airflow_executable }} scheduler -n ${SCHEDULER_RUNS} --pid /run/airflow/scheduler.pid
 Restart=always
 RestartSec=5s
 RuntimeDirectory=airflow

diff --git a/templates/airflow-webserver.service.j2 b/templates/airflow-webserver.service.j2
@@ -21,7 +21,7 @@ EnvironmentFile={{ airflow_environment_file_folder }}/airflow
 User={{ airflow_user }}
 Group={{ airflow_group }}
 Type=simple
-ExecStart=/usr/local/bin/airflow webserver --pid /run/airflow/webserver.pid
+ExecStart={{ airflow_executable }} webserver --pid /run/airflow/webserver.pid
 Restart=on-failure
 RestartSec=5s
 PrivateTmp=true

diff --git a/templates/airflow-worker.service.j2 b/templates/airflow-worker.service.j2
@@ -21,7 +21,7 @@ EnvironmentFile={{ airflow_environment_file_folder }}/airflow
 User={{ airflow_user }}
 Group={{ airflow_group }}
 Type=simple
-ExecStart=/usr/local/bin/airflow worker --pid /run/airflow/worker.pid
+ExecStart={{ airflow_executable }} worker --pid /run/airflow/worker.pid
 Restart=on-failure
 RestartSec=10s
 RuntimeDirectory=airflow