hdfs-spark-mongo-cluster.yml

version: "3.3"

services:

# jre
  jre-8:
    build: ./docker-files/jre/8u242-centos7
    image: jre:8u242-centos7
    container_name: jre-8
    networks:
      - spark-net

  jre:
    build: docker-files/jre/1-11.0.7.10-4-centos7
    image: jre:1-11.0.7.10-4-centos7
    container_name: jre
    networks:
      - spark-net

# Hadoop-2.7.7        
  base-2:
    build: ./docker-files/hadoop/2.7.7
    image: hadoop:2.7.7
    container_name: base-2
    networks:
      - spark-net

# Hadoop-3.2.1        
  base:
    build: ./docker-files/hadoop/3.2.1/base
    image: hadoop:3.2.1
    container_name: base
    networks:
      - spark-net

# spark-3.0.0
  spark:
    build: docker-files/spark/3.0.0
    image: spark:3.0.0
    container_name: spark
    networks:
      - spark-net        
       
# HDFS services
  namenode:
    build: ./docker-files/hadoop/3.2.1/namenode
    image: namenode:3.2.1
    container_name: namenode
    ports:
      - "9874:9870"
    volumes:
      - /etc/localtime:/etc/localtime:ro
      - ${VOLUMES_PATH}/hadoop/conf:/opt/hadoop-3.2.1/etc/hadoop
      - ${VOLUMES_PATH}/hadoop/logs:/opt/hadoop-3.2.1/logs
      - ${VOLUMES_PATH}/hadoop/namenode:/hadoop/dfs/name
    environment:
      CLUSTER_NAME: "BNDF"
      HADOOP_HEAPSIZE: "16384"
      HADOOP_CONF_DIR: "/etc/hadoop"
      HADOOP_LOG_DIR: "/opt/hadoop-3.2.1/logs"
      HADOOP_ROOT_LOGGER: "WARN,console,RFA"
      HADOOP_LOGFILE: "{{.Task.Name}}.log"
    restart: unless-stopped
    networks:
      - spark-net

  datanode:
    build: ./docker-files/hadoop/3.2.1/datanode
    image: datanode:3.2.1
    container_name: datanode
    volumes:
      - /etc/localtime:/etc/localtime:ro
      - ${VOLUMES_PATH}/hadoop/conf:/opt/hadoop-3.2.1/etc/hadoop
      - ${VOLUMES_PATH}/hadoop/logs:/opt/hadoop-3.2.1/logs
      - ${VOLUMES_PATH}/hadoop/datanode:/hadoop/dfs/data
    environment:
      SERVICE_PRECONDITION: "namenode:9870"
      HADOOP_HEAPSIZE: "16384"
      HADOOP_CONF_DIR: "/etc/hadoop"
      HADOOP_LOG_DIR: "/opt/hadoop-3.2.1/logs"
      HADOOP_ROOT_LOGGER: "WARN,console,RFA"
      HADOOP_LOGFILE: "{{.Task.Name}}.log"
    ports:
      - "9864:9864"      
    restart: unless-stopped
    networks:
      - spark-net

# HIVE services        
  hive-server:
    build: ./docker-files/docker-hive/
    image: hive:2.3.7
    container_name: hive-server
    volumes:
      - /etc/localtime:/etc/localtime:ro
      - ${VOLUMES_PATH}/hadoop/conf:/opt/hadoop-2.7.7/etc/hadoop
      - ${VOLUMES_PATH}/hive/conf:/opt/hive/conf
      - ${VOLUMES_PATH}/hive/logs:/opt/hive/logs
    environment:
      SERVICE_PRECONDITION: "hive-metastore:9083"
      HADOOP_HOME: "/opt/hadoop-2.7.7"
      HADOOP_HEAPSIZE: "16384"
      HADOOP_CONF_DIR: "/etc/hadoop"
      HIVE_CONF_DIR: "/opt/hive/conf"
      HADOOP_LOG_DIR: "/opt/hadoop-2.7.7/logs"
      HADOOP_ROOT_LOGGER: "WARN,console,RFA"
      HADOOP_LOGFILE: "{{.Task.Name}}.log"
    ports:
      - "10000:10000"
    restart: unless-stopped
    networks:
      - spark-net

  hive-init-db:
    build: ./docker-files/docker-hive/
    image: hive:2.3.7
    volumes:
      - /etc/localtime:/etc/localtime:ro
      - ${VOLUMES_PATH}/hadoop/conf:/opt/hadoop-2.7.7/etc/hadoop
      - ${VOLUMES_PATH}/hive/conf:/opt/hive/conf
      - ${VOLUMES_PATH}/hive/logs:/opt/hive/logs
    command: schematool -dbType postgres -initSchema
    networks:
      - spark-net        

  hive-metastore-postgresql:
    build: ./docker-files/docker-hive-metastore-postgresql/
    image: hive-metastore-postgresql:_-12
    container_name: hive-metastore-postgresql
    volumes:
      - ${VOLUMES_PATH}/postgres/data:/var/lib/postgresql/data
    ports:
      - "5432:5432"
    environment:        
      POSTGRES_USER: "postgres"
      POSTGRES_PASSWORD: "ns123"        
    restart: unless-stopped
    networks:
      - spark-net        
        
  hive-metastore:
    build: ./docker-files/docker-hive/
    image: hive:2.3.7
    container_name: hive-metastore
    volumes:
      - /etc/localtime:/etc/localtime:ro
      - ${VOLUMES_PATH}/hadoop/conf:/opt/hadoop-2.7.7/etc/hadoop
      - ${VOLUMES_PATH}/hive/conf:/opt/hive/conf
      - ${VOLUMES_PATH}/hive/logs:/opt/hive/logs
    command: hive --service metastore
    environment:
      SERVICE_PRECONDITION: "namenode:9870 datanode:9864 hive-metastore-postgresql:5432"
      HADOOP_HOME: "/opt/hadoop-2.7.7"
      HADOOP_HEAPSIZE: "16384"
      HADOOP_CONF_DIR: "/etc/hadoop"
      HIVE_CONF_DIR: "/opt/hive/conf"
      HADOOP_LOG_DIR: "/opt/hadoop-2.7.7/logs"
      HADOOP_ROOT_LOGGER: "WARN,console,RFA"
      HADOOP_LOGFILE: "{{.Task.Name}}.log"
    ports:
      - "9083:9083"
    restart: unless-stopped
    networks:
      - spark-net

# SPARK services    
  spark-master:
    image: spark:3.0.0
    container_name: spark-master
    ports:
      - 8080:8080
      - 7077:7077
    volumes:
      - ${VOLUMES_PATH}/spark/conf:/opt/spark-3.0.0-bin-hadoop3.2/conf
      - ${VOLUMES_PATH}/spark/logs:/opt/spark-3.0.0-bin-hadoop3.2/logs
      - ${VOLUMES_PATH}/spark/data:/data
      - ${VOLUMES_PATH}/spark/tmp:/tmp
      - ${DATA_PATH}:/sample-data
      - ./jars:/opt/jars
    environment:
      SPARK_NO_DAEMONIZE: "true"
    command: "sbin/start-master.sh"
    restart: unless-stopped
    networks:
      - spark-net

  spark-worker:
    image: spark:3.0.0
    container_name: spark-worker-1    
    volumes:
      - ${VOLUMES_PATH}/spark/conf:/opt/spark-3.0.0-bin-hadoop3.2/conf
      - ${VOLUMES_PATH}/spark/logs:/opt/spark-3.0.0-bin-hadoop3.2/logs
      - ${DATA_PATH}:/sample-data
      - ${VOLUMES_PATH}/spark/localdir:/data/localdir
      - ${VOLUMES_PATH}/spark/workerdir:/data/workerdir
      - ${VOLUMES_PATH}/spark/tmp:/tmp
      - ./jars:/opt/jars
    environment:
      SPARK_NO_DAEMONIZE: "true"
    command: "sbin/start-slave.sh spark://spark-master:7077"        
    restart: unless-stopped
    networks:
      - spark-net


# Apache ZEPPELIN
  zeppelin:
    build: ./docker-files/zeppelin/
    image: zeppelin:0.9.0-preview2
    container_name: zeppelin    
    ports:
      - 8085:8080
      - 4040:4040  
    volumes:
      - ${VOLUMES_PATH}/zeppelin/conf:/opt/zeppelin/conf
      - ${VOLUMES_PATH}/zeppelin/notebook:/opt/zeppelin/notebook
      - ${VOLUMES_PATH}/zeppelin/logs:/opt/zeppelin/logs
      - ${DATA_PATH}:/sample-data
      - ${VOLUMES_PATH}/spark/conf:/opt/spark-3.0.0-bin-hadoop3.2/conf
      - ${VOLUMES_PATH}/spark/localdir:/data/localdir
      - ${VOLUMES_PATH}/spark/workerdir:/data/workerdir
      - ./jars:/opt/jars
    restart: unless-stopped
    networks:
      - spark-net
        
# MongoDB
  mongo:
    image: mongo:4.2.6
    container_name: mongos
    hostname: mongos
    environment:
      MONGO_INITDB_ROOT_USERNAME: root
      MONGO_INITDB_ROOT_PASSWORD: ns123
    volumes:
      - ${VOLUMES_PATH}/mongo/data:/data/db
      - ${VOLUMES_PATH}/mongo/logs:/logs
    ports:
      - target: 27017
        published: 27017
        protocol: tcp
        mode: host        
    command: "mongod --bind_ip_all"
    restart: unless-stopped
    networks:
      - spark-net

# Netdata
  netdata:
    image: netdata/netdata
    container_name: netdata
    hostname: netdata
    ports:
      - 19999:19999
    cap_add:
      - SYS_PTRACE
    security_opt:
      - apparmor:unconfined
    volumes:
      - /etc/passwd:/host/etc/passwd:ro
      - /etc/group:/host/etc/group:ro
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
    restart: unless-stopped
    networks:
      - spark-net
    

networks:
  spark-net:
    external:
     name: spark-net