Merge pull request #38 from zbrookle/fix_installation

BUG: Fix installation

MANIFEST.in

@@ -5,5 +5,6 @@ include avionix_airflow/docker/airflow/Dockerfile
 include avionix_airflow/docker/airflow/script/entrypoint.sh
 include avionix_airflow/kubernetes/monitoring/dashboards/grafana_dashboard.json
 include avionix_airflow/kubernetes/monitoring/event_js/event.js
+include requirements.txt
 
 

avionix_airflow/kubernetes/airflow/dag_retrieval.py

@@ -16,7 +16,10 @@ def __init__(self, airflow_options: AirflowOptions, cloud_options: CloudOptions)
                 JobTemplateSpec(
                     JobSpec(
                         PodTemplateSpec(
-                            AirflowMeta("sync-dags-pod",),
+                            AirflowMeta(
+                                "sync-dags-pod",
+                                annotations=cloud_options.dag_retrieval_annotations,
+                            ),
                             PodSpec(
                                 [
                                     Container(

avionix_airflow/kubernetes/cloud/aws/aws_options.py

@@ -30,6 +30,7 @@ class AwsOptions(CloudOptions):
     :param external_dns_role_arn: The IAM role for setting the domain name
     :param autoscaling_role_arn: The IAM role for controlling the cluster scaling
     :param domain: The AWS domain name to use
+    :param dag_sync_role_arn: The IAM role use for controlling retrieval of dags
     :param domain_filters: A list used to filter out route53 domain names
     :param use_ssl: Whether or not to use SSL encryption, (Recommended to be True)
     """
@@ -48,6 +49,7 @@ def __init__(
         alb_role_arn: str,
         external_dns_role_arn: str,
         autoscaling_role_arn: str,
+        dag_sync_role_arn: str,
         domain: str,
         domain_filters: Optional[List[str]] = None,
         use_ssl: bool = False,
@@ -63,11 +65,9 @@ def __init__(
         self.__values = ValueOrchestrator()
         self.__use_ssl = use_ssl
         self.__autoscaling_role_arn = autoscaling_role_arn
+        self.__dag_sync_role_arn = dag_sync_role_arn
         super().__init__(
-            StorageClass(
-                ObjectMeta(name="efs-sc"), None, None, None, "efs.csi.aws.com", None
-            ),
-            "Filesystem",
+            StorageClass(ObjectMeta(name="efs-sc"), "efs.csi.aws.com"), "Filesystem",
         )
 
     def get_csi_persistent_volume_source(self, name: str):
@@ -238,3 +238,7 @@ def webserver_service_annotations(self) -> Dict[str, str]:
             "alb.ingress.kubernetes.io/healthcheck-path": "/airflow",
             "alb.ingress.kubernetes.io/successCodes": "200,308",
         }
+
+    @property
+    def dag_retrieval_annotations(self) -> Dict[str, str]:
+        return {"iam.amazonaws.com/role": self.__dag_sync_role_arn}

avionix_airflow/kubernetes/cloud/cloud_options.py

@@ -65,3 +65,8 @@ def get_elastic_search_proxy_elements(
     @abstractmethod
     def webserver_service_annotations(self) -> Dict[str, str]:
         return {}
+
+    @property
+    @abstractmethod
+    def dag_retrieval_annotations(self) -> Dict[str, str]:
+        return {}

avionix_airflow/kubernetes/cloud/local/local_options.py

@@ -57,3 +57,7 @@ def webserver_service_annotations(self) -> Dict[str, str]:
     @property
     def extra_ingress_paths(self) -> List[AirflowIngressPath]:
         return []
+
+    @property
+    def dag_retrieval_annotations(self) -> Dict[str, str]:
+        return {}

avionix_airflow/kubernetes/monitoring/elasticsearch_dependency.py

@@ -1,7 +1,13 @@
 from avionix import ChartDependency
+from avionix.kube.core import PersistentVolumeClaimSpec, ResourceRequirements
 
 
 class ElasticSearchDependency(ChartDependency):
+
+    _labels = {"storage-type": "elasticsearch"}
+    _storage = {"storage": "100Mi"}
+    _access_modes = ["ReadWriteOnce"]
+
     def __init__(self):
         super().__init__(
             "elasticsearch",
@@ -17,11 +23,11 @@ def __init__(self):
                     "requests": {"cpu": "100m", "memory": "512M"},
                     "limits": {"cpu": "1000m", "memory": "512M"},
                 },
-                "volumeClaimTemplate": {
-                    "accessModes": ["ReadWriteOnce"],
-                    "storageClassName": "standard",
-                    "resources": {"requests": {"storage": "100M"}},
-                },
+                "volumeClaimTemplate": PersistentVolumeClaimSpec(
+                    access_modes=self._access_modes,
+                    resources=ResourceRequirements(requests=self._storage),
+                    storage_class_name="standard",
+                ).to_dict(),
                 "service": {"type": "NodePort", "nodePort": 30012},
             },
         )

avionix_airflow/tests/conftest.py

@@ -1,3 +1,5 @@
+from subprocess import check_call
+
 from avionix.testing.installation_context import ChartInstallationContext
 import pytest
 
@@ -81,6 +83,10 @@ def build_chart(airflow_options, sql_options, redis_options, monitoring_options)
     builder = get_chart_builder(
         airflow_options, sql_options, redis_options, monitoring_options
     )
+    if kubectl_name_dict("persistentvolume"):
+        # Stateful Sets do not automatically clean up their pvc templates so this
+        # must be done manually
+        check_call("kubectl delete pvc -n airflow --all".split(" "))
     while True:
         if not filter_out_pvc(kubectl_name_dict("persistentvolume")):
             break

environment.yml

@@ -14,6 +14,6 @@ dependencies:
   - pytest-rerunfailures
   - pip:
     # Main requirements
-    - avionix==0.3.1
+    - avionix==0.4.0
     - python-hosts==1.0.0
     - cryptography==3.0

examples/aws_example.py

@@ -25,14 +25,15 @@
         create_database_in_cluster=False,
     ),
     cloud_options=AwsOptions(
-        "fs-12345",
-        "my-cluster",
-        "arn:aws:iam::123456789012:role/es-role",
-        "arn:aws:iam::123456789012:role/default-role",
-        "arn:aws:iam::123456789012:role/alb-role",
-        "arn:aws:iam::123456789012:role/external-dns-role",
-        "arn:aws:iam::123456789012:role/autoscaling-role:",
-        "my.airflow.domain.com",
+        efs_id="fs-12345",
+        cluster_name="my-cluster",
+        elastic_search_access_role_arn="arn:aws:iam::123456789012:role/es-role",
+        default_role_arn="arn:aws:iam::123456789012:role/default-role",
+        alb_role_arn="arn:aws:iam::123456789012:role/alb-role",
+        external_dns_role_arn="arn:aws:iam::123456789012:role/external-dns-role",
+        autoscaling_role_arn="arn:aws:iam::123456789012:role/autoscaling-role",
+        domain="my.airflow.domain.com",
+        dag_sync_role_arn="arn:aws:iam::123456789012:role/dag_sync_role",
         use_ssl=True,
     ),
 )

requirements.txt

@@ -1,3 +1,3 @@
-avionix>=0.3.1
+avionix>=0.4.0
 python-hosts==1.0.0
 cryptography==3.0