docker-compose-sparql-web.yml

version: "2"

services:
  #debugger:
  #  image: sjourdan/toolbox
  #  container_name: debugger
  #  networks:
  #    - squirrel:latest
  #  dns: 8.8.8.8
  #  command: nc -l 50070

  ##configure ckan to open port of python end point
  #ckan:
  #  image: python:latest
  #  container_name: ckan
  #  networks:
  #    - squirrel:latest
  #  command: python rabbitrecieve.py
  #  ports:

  frontier:
    image: squirrel:latest
    container_name: frontier
    environment:
      - HOBBIT_RABBIT_HOST=rabbit
      - SEED_FILE=/var/squirrel/seeds.txt
      - URI_WHITELIST_FILE=/var/squirrel/whitelist.txt
      - RDB_HOST_NAME=rethinkdb
      - RDB_PORT=28015
      - COMMUNICATION_WITH_WEBSERVICE=true
      - VISUALIZATION_OF_CRAWLED_GRAPH=true
    volumes:
      - ./data/frontier:/var/squirrel/data
      - ./seed/seeds.txt:/var/squirrel/seeds.txt:ro
      - ./whitelist/whitelist.txt:/var/squirrel/whitelist.txt:ro
    command: java -cp squirrel.jar org.hobbit.core.run.ComponentStarter org.aksw.simba.squirrel.components.FrontierComponent

  web:
    image: squirrel/webimage:latest
    container_name: web
    ports:
      - "8080:8080"

  sparqlhost:
    image: stain/jena-fuseki
    container_name: sparqlhost
    ports:
      - "3030:3030"
    volumes:
      - ./data/sparqlhost/sparqlhost_data:/fuseki
    environment:
      - ADMIN_PASSWORD=pw123
      - JVM_ARGS=-Xmx2g

  rethinkdb:
    image: rethinkdb:2.3.5
    volumes:
      - ./data/rethinkdb:/data
    ports:
      - "28015:28015"

  # message bus
  rabbit:
    image: rabbitmq:management
    container_name: rabbit
    hostname: rabbit
    ports:
      - "8081:15672"
      # Forwarding the port for testing
      - "5672:5672"

  worker1:
    image: squirrel:latest
    container_name: worker1
    environment:
      - HOBBIT_RABBIT_HOST=rabbit
      - OUTPUT_FOLDER=/var/squirrel/data
      - HTML_SCRAPER_YAML_PATH=/var/squirrel/yaml
      - CONTEXT_CONFIG_FILE=/var/squirrel/spring-config/context-sparqlStoreBased.xml
      - SPARQL_HOST_NAME=sparqlhost
      #- CKAN_WHITELIST_FILE=/var/squirrel/ckanwhitelist.txt
      - SPARQL_HOST_PORT=3030
      - DEDUPLICATION_ACTIVE=true
      - RDB_HOST_NAME=rethinkdb
      - RDB_PORT=28015
      #-CKAN_PORT=
    volumes:
        - ./data/worker1:/var/squirrel/data
        - ./yaml:/var/squirrel/yaml
        - ./spring-config:/var/squirrel/spring-config
        #- ./whitelist/ckanwhitelist.txt:/var/squirrel/ckanwhitelist.txt:ro
    command: java -cp squirrel.jar org.aksw.simba.squirrel.components.WorkerComponentStarter

  worker2:
    image: squirrel:latest
    container_name: worker2
    environment:
      - HOBBIT_RABBIT_HOST=rabbit
      - OUTPUT_FOLDER=/var/squirrel/data
      - HTML_SCRAPER_YAML_PATH=/var/squirrel/yaml
      - CONTEXT_CONFIG_FILE=/var/squirrel/spring-config/context-sparqlStoreBased.xml
      - SPARQL_HOST_NAME=sparqlhost
      #- CKAN_WHITELIST_FILE=/var/squirrel/ckanwhitelist.txt
      - SPARQL_HOST_PORT=3030
      - DEDUPLICATION_ACTIVE=true
      - RDB_HOST_NAME=rethinkdb
      - RDB_PORT=28015
      #-CKAN_PORT=
    volumes:
        - ./data/worker2:/var/squirrel/data
        - ./yaml:/var/squirrel/yaml
        - ./spring-config:/var/squirrel/spring-config
        #- ./whitelist/ckanwhitelist.txt:/var/squirrel/ckanwhitelist.txt:ro
    command: java -cp squirrel.jar org.aksw.simba.squirrel.components.WorkerComponentStarter

  worker3:
    image: squirrel:latest
    container_name: worker3
    environment:
      - HOBBIT_RABBIT_HOST=rabbit
      - OUTPUT_FOLDER=/var/squirrel/data
      - HTML_SCRAPER_YAML_PATH=/var/squirrel/yaml
      - CONTEXT_CONFIG_FILE=/var/squirrel/spring-config/context-sparqlStoreBased.xml
      - SPARQL_HOST_NAME=sparqlhost
      #- CKAN_WHITELIST_FILE=/var/squirrel/ckanwhitelist.txt
      - SPARQL_HOST_PORT=3030
      - DEDUPLICATION_ACTIVE=true
      - RDB_HOST_NAME=rethinkdb
      - RDB_PORT=28015
      #-CKAN_PORT=
    volumes:
        - ./data/worker3:/var/squirrel/data
        - ./yaml:/var/squirrel/yaml
        - ./spring-config:/var/squirrel/spring-config
        #- ./whitelist/ckanwhitelist.txt:/var/squirrel/ckanwhitelist.txt:ro
    command: java -cp squirrel.jar org.aksw.simba.squirrel.components.WorkerComponentStarter

  deduplicator:
    image: squirrel
    container_name: deduplicator
    environment:
      DEDUPLICATION_ACTIVE: "true"
      HOBBIT_RABBIT_HOST: rabbit
      OUTPUT_FOLDER: /var/squirrel/data
      RDB_HOST_NAME: rethinkdb
      RDB_PORT: 28015
      SPARQL_HOST_NAME: sparqlhost
      SPARQL_HOST_PORT: 3030
      SERVICE_PRECONDITION: "rethinkdb:28015 rabbit:5672"
    volumes:
        - ./data/deduplicator:/var/squirrel/data
    command: java -cp squirrel.jar org.hobbit.core.run.ComponentStarter org.aksw.simba.squirrel.components.DeduplicatorComponent