diff --git a/airflow/airflow-db.hcl b/airflow/airflow-db.hcl new file mode 100644 index 0000000..9e88999 --- /dev/null +++ b/airflow/airflow-db.hcl @@ -0,0 +1,130 @@ +job "airflow-db" { + datacenters = ["dc1"] + type = "service" + + update { + max_parallel = 1 + min_healthy_time = "10s" + healthy_deadline = "5m" + progress_deadline = "10m" + auto_revert = false + canary = 0 + } + + affinity { + attribute = "${node.unique.name}" + value = "client-0" + weight = 100 + } + + group "postgres" { + count = 1 + + restart { + attempts = 2 + interval = "10m" + delay = "15s" + mode = "fail" + } + + network{ + port "postgres"{ + static = 5432 + to = 5432 + } + } + service { + name = "postgres" + port = "postgres" + # provider="nomad" + # tags = [ + # "traefik.enable=true", + # "traefik.tcp.routers.postgres.rule=HostSNI(`*`)", + # "traefik.tcp.routers.postgres.entryPoints=postgres", + # "traefik.tcp.routers.postgres.service=postgres", + # # services (needed for TCP) + # "traefik.tcp.services.postgres.loadbalancer.server.port=5432" + # ] + check { + type = "tcp" + port = "postgres" + interval = "10s" + timeout = "2s" + } + } + + task "postgres" { + driver = "docker" + env { + POSTGRES_USER= "airflow" + POSTGRES_PASSWORD= "airflow" + POSTGRES_DB= "airflow" + } + + config { + ports=["postgres"] + image = "postgres:13" + network_mode="host" + volumes=[ + "local/postgres-db-volume:/var/lib/postgresql/data" + ] + + } + + # resources { + # cpu = 100 + # memory = 1024 + # } + } + } + + group "redis" { + count = 1 + + network{ + port "redis"{ + static = 6379 + to = 6379 + } + } + restart { + attempts = 2 + interval = "10m" + delay = "15s" + mode = "fail" + } + service { + name = "redis" + port = "redis" + # provider="nomad" + + # tags = [ + # # routers + # "traefik.tcp.routers.redis.rule=HostSNI(`*`)", + # "traefik.tcp.routers.redis.entryPoints=redis", + # "traefik.tcp.routers.redis.service=redis", + # # services (needed for TCP) + # "traefik.tcp.services.redis.loadbalancer.server.port=6379" + # ] + check { + type = "tcp" + port = "redis" + interval = "10s" + timeout = "2s" + } + } + + task "redis" { + driver = "docker" + config { + image = "redis:6" + network_mode="host" + ports=["redis"] + } + # resources { + # cpu = 100 + # memory = 1024 + # } + } + } +} diff --git a/airflow/airflow.hcl b/airflow/airflow.hcl new file mode 100644 index 0000000..4244e59 --- /dev/null +++ b/airflow/airflow.hcl @@ -0,0 +1,432 @@ +variable "conf" { + default = "https://gist.githubusercontent.com/darkslategrey/37620fab3f5922240d73ba8185519176/raw/02552253e3630ea55e7e52fc70756285cfdc2875/gistfile1.txt" +} + +job "airflow" { + datacenters = ["dc1"] + type = "service" + + meta { + deploy = uuidv4() + } + + update { + max_parallel = 1 + min_healthy_time = "10s" + healthy_deadline = "5m" + progress_deadline = "10m" + auto_revert = false + canary = 0 + } + + constraint { + attribute = "${node.unique.name}" + value = "client-1" + operator = "=" + # weight = 100 + } + + group "airflow" { + + volume "airflow" { + type = "host" + source = "airflow" + read_only = false + } + volume "airflow-inputs" { + type = "host" + source = "airflow-inputs" + read_only = false + } + volume "csi" { + type = "host" + source = "csi" + read_only = false + } + + network{ + dns { + servers = ["${attr.unique.network.ip-address}"] + } + port "afwebserver"{ + #static = 8080 + to = 8080 + } + } + + task "chown-opt-airflow" { + lifecycle { + hook = "prestart" + sidecar = false + } + volume_mount { + volume = "airflow" + destination = "/opt/airflow" + read_only = false + } + driver = "docker" + user = "root" + template { + data = <