docker-compose.yml

# Verify volumes
version: "3.8"
services:  
    # Postgres used by Airflow
    postgres:
        image: postgres:13.6
        networks:
            - default_net
        environment:
            - POSTGRES_USER=airflow
            - POSTGRES_PASSWORD=airflow
            - POSTGRES_DB=airflow
        volumes:
            - "./src/postgres:/var/lib/postgresql/data 777"
        ports:
            - "5432:5432"
    # Airflow LocalExecutor
    airflow-webserver:
        image: mutual_fund_recommendation_etl:latest
        restart: always
        networks:
            - default_net
        depends_on:
            - postgres
            - mysql
        environment:
            - LOAD_EX=n
            - AIRFLOW__CORE__LOAD_EXAMPLES=false
            - EXECUTOR=Local
        volumes:
            - ./src/dags:/usr/local/airflow/dags # DAGs folder
            - ./src/logs:/usr/local/airflow/logs # logs folder
            - ./src/app:/usr/local/spark/app # app folder
            - ./src/resources:/usr/local/spark/resources # Spark Resources (same path in airflow and spark)
        ports:
            - "8080:8080" #host:container
        command: webserver
        healthcheck:
            test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"]
            interval: 30s
            timeout: 30s
            retries: 3

    # Spark with N workers
    spark-master:
        image: bitnami/spark:3.2.1
        #user: root # Run container as root container: https://docs.bitnami.com/tutorials/work-with-non-root-containers/
        hostname: spark
        networks:
            - default_net
        environment:
            - SPARK_MODE=master
            - SPARK_RPC_AUTHENTICATION_ENABLED=no
            - SPARK_RPC_ENCRYPTION_ENABLED=no
            - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
            - SPARK_SSL_ENABLED=no
        volumes:
            - ./src/app:/usr/local/spark/app # Scripts (same path in airflow and spark)
            - ./src/resources:/usr/local/spark/resources # Resources (same path in airflow and spark)
        ports:
            - "8081:8080"
            - "7077:7077"
        depends_on:
            - mysql

    spark-worker:
        image: bitnami/spark:3.2.1
        #user: root
        networks:
            - default_net
        environment:
            - SPARK_MODE=worker
            - SPARK_MASTER_URL=spark://spark:7077
            - SPARK_WORKER_MEMORY=1G
            - SPARK_WORKER_CORES=1
            - SPARK_RPC_AUTHENTICATION_ENABLED=no
            - SPARK_RPC_ENCRYPTION_ENABLED=no
            - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
            - SPARK_SSL_ENABLED=no
        depends_on:
            - mysql
        volumes:
            - ./src/app:/usr/local/spark/app # Scripts (same path in airflow and spark)
            - ./src/resources:/usr/local/spark/resources # Resources (same path in airflow and spark)

    redis:
        image: redis:3.2
        restart: always
        ports:
            - 6379:6379
        volumes:
            - redis:/data
    mysql:
        image: mysql:5.7
        restart: always
        environment:
            MYSQL_USER: 'superset'
            MYSQL_PASSWORD: 'superset'
            MYSQL_DATABASE: 'superset'
            MYSQL_ROOT_PASSWORD: 'root'
        ports:
            - 3306:3306
        volumes:
            - mysql:/var/lib/mysql
    superset:
        image: amancevice/superset:latest
        restart: always
        environment:
            MYSQL_USER: 'superset'
            MYSQL_PASS: 'superset'
            MYSQL_DATABASE: 'superset'
            MYSQL_HOST: 'mysql'
            MYSQL_PORT: 3306
            REDIS_HOST: 'redis'
            REDIS_PORT: 6379
        ports:
            - 8088:8088
            - 5555:5555
        depends_on:
            - mysql
            - redis
        volumes:
            - ./src/app/configs/superset_config.py:/etc/superset/superset_config.py
     # Jupyter Notebooks
    jupyter-pyspark:
        # image: jupyter/pyspark-notebook:spark-3.2.1 # pyspark
        image: jupyter/pyspark-notebook:python-3.8.8 # pyspark
        #image: jupyter/all-spark-notebook:spark-3.2.1 # scala
        #image: jupyter/datascience-notebook:latest # julia 
        networks:
            - default_net
        ports:
          - "8888:8888"
        volumes:
          - ./src/resources/notebooks:/usr/local/spark/resources/notebooks/
volumes:
  mysql:
    external: false
  redis:
    external: false
networks:
    default_net: