#50 Fix Airflow extra packages scenario and compatibility with more A…

…nsible versions

.travis.yml

@@ -4,16 +4,18 @@ python: "2.7.15"
 sudo: required
 
 env:
-  - PIPENV_IGNORE_VIRTUALENVS=1
+  - ansible_version: 2.4.5
+  - ansible_version: 2.7.9
+  - ansible_version: 2.8.1
 
 services:
   - docker
 
 install:
-  - pip install pipenv==2018.11.26
-  - pipenv install -r test-requirements.txt
+  - pip install ansible==${ansible_version}
+  - pip install -r test-requirements.txt
 script:
-  - pipenv run molecule test
+  - molecule test --all
 
 notifications:
   webhooks: https://galaxy.ansible.com/api/v1/notifications/

CHANGELOG.md

@@ -7,7 +7,7 @@ This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a ch
 ## [1.7.4](https://github.com/idealista/airflow-role/tree/1.7.4)
 [Full Changelog](https://github.com/idealista/airflow-role/compare/1.7.3...1.7.4)
 ### Fixed
-- *[#47](https://github.com/idealista/airflow-role/issues/50) Fix deprecation warning from jinja templates @adrimarteau
+- *[#50](https://github.com/idealista/airflow-role/issues/50) Fix deprecation warning from jinja templates @adrimarteau @jnogol
 
 ## [1.7.3](https://github.com/idealista/airflow-role/tree/1.7.3)
 [Full Changelog](https://github.com/idealista/airflow-role/compare/1.7.2...1.7.3)

README.md

@@ -72,6 +72,12 @@ airflow_required_python_packages:
   - {name: pyasn1, version: 0.4.4}
 ```
 
+`airflow_extra_packages` (available at: https://airflow.apache.org/installation.html#extra-packages) should be a list following this format:
+``` yml
+airflow_extra_packages:
+  - celery
+  - mysql
+```
 ## Testing
 
 ```

defaults/main.yml

@@ -1,6 +1,8 @@
 ---
 ## General
 airflow_version: 1.9.0
+# Available extra packages: https://airflow.apache.org/installation.html#extra-packages
+# List should follow Ansible's YAML basics: https://docs.ansible.com/ansible/latest/reference_appendices/YAMLSyntax.html#yaml-basics
 airflow_extra_packages: []
 
 airflow_pip_executable: "pip"

molecule/default/verify.yml

@@ -7,7 +7,7 @@
   hosts: all
   become: true
   vars:
-    goss_version: v0.3.6
+    goss_version: v0.3.7
     goss_arch: amd64
     goss_dst: /usr/local/bin/goss
     goss_url: "https://github.com/aelsabbahy/goss/releases/download/{{ goss_version }}/goss-linux-{{ goss_arch }}"
@@ -40,7 +40,7 @@
       register: test_files
 
     - name: Execute Goss tests
-      command: "{{ goss_dst }} -g {{ item }} validate --format {{ goss_format }} --retry-timeout 2s"
+      command: "{{ goss_dst }} -g {{ item }} validate --format {{ goss_format }} --retry-timeout 10s"
       register: test_results
       with_items: "{{ test_files.stdout_lines }}"
 

molecule/extra_packages/Dockerfile.j2

@@ -0,0 +1,9 @@
+# Molecule managed
+
+{% if item.registry is defined %}
+FROM {{ item.registry.url }}/{{ item.image }}
+{% else %}
+FROM {{ item.image }}
+{% endif %}
+
+RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get upgrade -y && apt-get install -y python sudo bash ca-certificates systemd systemd-sysv && apt-get clean; fi

molecule/extra_packages/files/sample-dag.py

@@ -0,0 +1,33 @@
+from datetime import datetime
+from airflow import DAG
+from airflow.operators.dummy_operator import DummyOperator
+
+args = {
+    'owner': 'airflow',
+    'start_date': datetime(2017, 9, 10, 0, 0),
+    'random_logic': False
+}
+
+dag = DAG(
+    'sample_dag',
+    schedule_interval="@once",
+    default_args=args
+)
+
+t1 = DummyOperator(
+    task_id='extract_data',
+    dag=dag
+)
+
+t2 = DummyOperator(
+    task_id='load_data',
+    dag=dag
+)
+
+t3 = DummyOperator(
+    task_id='random_task',
+    dag=dag
+)
+
+t1.set_downstream(t2)
+t2.set_downstream(t3)

molecule/extra_packages/group_vars/airflow/main.yml

@@ -0,0 +1,15 @@
+---
+airflow_fernet_key: xKy13nPFfDflJ0DYGVTwf_DEmbItfURHlEDxrt-bBQw=
+
+airflow_required_libs:
+  - acl
+  - python-pip
+  - python-mysqldb
+
+airflow_extra_packages:
+  - celery
+  - mysql
+
+# airflow_extra_packages: celery, mysql
+
+# airflow_extra_packages: [celery, mysql]

molecule/extra_packages/molecule.yml

@@ -0,0 +1,33 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: docker
+lint:
+  name: yamllint
+  enabled: False
+
+platforms:
+  - name: airflow_extra_packages-${MOLECULE_DISTRO:-stretch}
+    image: debian:${MOLECULE_DISTRO:-stretch}
+    privileged: true
+    capabilities:
+      - SYS_ADMIN
+    volumes:
+      - '/sys/fs/cgroup:/sys/fs/cgroup:ro'
+    groups:
+      - airflow
+    command: '/lib/systemd/systemd'
+
+provisioner:
+  name: ansible
+  lint:
+    name: ansible-lint
+    enabled: False
+scenario:
+  name: extra_packages
+verifier:
+  name: goss
+  lint:
+    name: yamllint
+    enabled: False

molecule/extra_packages/playbook.yml

@@ -0,0 +1,6 @@
+---
+
+- name: Converge
+  hosts: all
+  roles:
+    - role: airflow-role

molecule/extra_packages/tests/test_airflow.yml

@@ -0,0 +1,24 @@
+service:
+{% for airflow_service in airflow_services %}
+{% if airflow_services[airflow_service]["enabled"] %}
+  {{ airflow_service }}:
+    enabled: true
+    running: true
+{% endif %}
+{% endfor %}
+
+user:
+  {{ airflow_user }}:
+    exists: true
+    groups:
+      - {{ airflow_group }}
+
+group:
+  {{ airflow_group }}:
+    exists: true
+
+command:
+  /usr/local/bin/airflow version:
+    exit-status: 0
+    stdout:
+      - "{{ airflow_version }}"

molecule/extra_packages/verify.yml

@@ -0,0 +1,56 @@
+---
+# This is an example playbook to execute goss tests.
+# Tests need distributed to the appropriate ansible host/groups
+# prior to execution by `goss validate`.
+
+- name: Verify
+  hosts: all
+  become: true
+  vars:
+    goss_version: v0.3.7
+    goss_arch: amd64
+    goss_dst: /usr/local/bin/goss
+    goss_url: "https://github.com/aelsabbahy/goss/releases/download/{{ goss_version }}/goss-linux-{{ goss_arch }}"
+    goss_test_directory: /tmp
+    goss_format: documentation
+
+  vars_files:
+    - ../../defaults/main.yml
+    - ./group_vars/airflow/main.yml
+
+  tasks:
+    - name: Download and install Goss
+      get_url:
+        url: "{{ goss_url }}"
+        dest: "{{ goss_dst }}"
+        mode: 0755
+      register: download_goss
+      until: download_goss is succeeded
+      retries: 3
+
+    - name: Copy Goss tests to remote
+      template:
+        src: "{{ item }}"
+        dest: "{{ goss_test_directory }}/{{ item | basename }}"
+      with_fileglob:
+        - "{{ lookup('env', 'MOLECULE_VERIFIER_TEST_DIRECTORY') }}/test_*.yml"
+
+    - name: Register test files
+      shell: "ls {{ goss_test_directory }}/test_*.yml"
+      register: test_files
+
+    - name: Execute Goss tests
+      command: "{{ goss_dst }} -g {{ item }} validate --format {{ goss_format }} --retry-timeout 10s"
+      register: test_results
+      with_items: "{{ test_files.stdout_lines }}"
+
+    - name: Display details about the Goss results
+      debug:
+        msg: "{{ item.stdout_lines }}"
+      with_items: "{{ test_results.results }}"
+
+    - name: Fail when tests fail
+      fail:
+        msg: "Goss failed to validate"
+      when: item.rc != 0
+      with_items: "{{ test_results.results }}"

tasks/install.yml

@@ -67,12 +67,23 @@
   environment:
     SLUGIFY_USES_TEXT_UNIDECODE: "yes"
 
+- name: Airflow | Installing Airflow Extra Packages (prior to Ansible 2.7)
+  pip:
+    executable: "{{ airflow_pip_executable }}"
+    name: "apache-airflow[{{ item }}]=={{ airflow_version }}"
+  with_items: "{{ airflow_extra_packages }}"
+  when: 
+    - airflow_extra_packages
+    - ansible_version.full is version_compare('2.7', '<')
+
 - name: Airflow | Installing Airflow Extra Packages
   pip:
     executable: "{{ airflow_pip_executable }}"
-    name: apache-airflow[{{ airflow_extra_packages }}]
+    name: "apache-airflow[{{ airflow_extra_packages | join(', ') }}]"
     version: "{{ airflow_version }}"
-  when: airflow_extra_packages
+  when: 
+    - airflow_extra_packages
+    - ansible_version.full is version_compare('2.7', '>=')
 
 - name: Airflow | Installing DAGs dependencies
   pip:

test-requirements.txt

@@ -1,3 +1,2 @@
-ansible==2.4.5.0
 molecule==2.20.1
 docker==3.7.2