docker-compose.yml

version: '2'

services:
  kafka_monitor:
    # image: istresearch/scrapy-cluster:kafka-monitor-1.2.1
    build:
      context: ./
      dockerfile: ./docker/kafka-monitor/Dockerfile
    depends_on:
      - kafka
      - redis
    restart: always
    volumes:
     - /vol_b/data/scrapy_cluster_data:/var/lib/crawl_dump
  kafka_dump:
    # image: istresearch/scrapy-cluster:kafka-monitor-1.2.1
    build:
      context: ./
      dockerfile: ./docker/kafka-monitor/Dockerfile_dump
    depends_on:
      - kafka
      - redis
    restart: always
    volumes:
     - /vol_b/data/scrapy_cluster_data:/var/lib/crawl_dump
  redis_monitor:
    image: istresearch/scrapy-cluster:redis-monitor-1.2.1
    depends_on:
      - kafka
      - redis
      - zookeeper
    restart: always
  crawler:
    # image: istresearch/scrapy-cluster:crawler-1.2.1
    build:
      context: ./
      dockerfile: ./docker/crawler/Dockerfile
    depends_on:
      - kafka
      - redis
      - zookeeper
    restart: always
    privileged: true
  rest:
    image: istresearch/scrapy-cluster:rest-1.2.1
    depends_on:
      - kafka
      - redis
    restart: always
    ports:
      - "5343:5343"
  redis:
    image: redis
    ports:
      - "6379"
    restart: always
  zookeeper:
    image: zookeeper
    ports:
      - "2181:2181"
    restart: always
  kafka:
    image: wurstmeister/kafka
    ports:
      - "9092:9092"
    environment:
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_ADVERTISED_HOST_NAME: kafka
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
    depends_on:
      - zookeeper
    restart: always
#  chrome:
#    image: selenium/node-chrome:3.9.1-actinium
#    volumes:
#      - /dev/shm:/dev/shm
#    depends_on:
#      - hub
#    environment:
#      HUB_HOST: hub
#  hub:
#    image: selenium/hub:3.9.1-actinium
#    ports:
#      - "4444"