docker-compose.yml

version: "3.4"
name: "database-replication"

services:
  source_db:
    image: postgres:14-alpine
    container_name: "source_db"
    ports:
      - "5432:5432"
    volumes:
      - .docker/data/postgres:/var/lib/postgresql/pgdata
      - .docker/backups/postgres:/backups
      - .docker/postgres-add-de-user.sh:/docker-entrypoint-initdb.d/postgres-add-de-user.sh
    environment:
      - POSTGRES_USER=${POSTGRES_USER}
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
      - POSTGRES_DB=${POSTGRES_DB}
      - PGDATA=/var/lib/postgresql/pgdata
    restart: unless-stopped
    networks:
      - upstream_networks
      - kafka_networks
    healthcheck:
      test: pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 10s

  adminer:
    image: adminer:standalone
    container_name: "adminer"
    ports:
      - "8080:8080"
    depends_on:
      - source_db
    networks:
      - upstream_networks
    restart: unless-stopped

  upstream-app:
    build:
      context: .
      dockerfile: .docker/images/app/Dockerfile
    container_name: "upstream-app"
    ports:
      - "8501:8501"
    volumes:
      - ./app:/app
    environment:
      - POSTGRES_USER=${POSTGRES_USER}
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
      - POSTGRES_HOST=source_db
      - POSTGRES_PORT=${POSTGRES_PORT}
      - POSTGRES_DB=${POSTGRES_DB}
    restart: on-failure
    depends_on:
      - source_db
    networks:
      - upstream_networks
    healthcheck:
      test: curl --fail http://localhost:8501/healthz || exit 1
      interval: 30s
      timeout: 30s
      retries: 5
      start_period: 10s

  zookeeper:
    image: bitnami/zookeeper:3.8
    container_name: zookeeper
    volumes:
      - .docker/data/zookeeper:/bitnami/zookeeper
    environment:
      - ZOO_SERVER_ID=0
      - ALLOW_ANONYMOUS_LOGIN=yes
      # How far out of date a server can be from a leader
      - ZOO_SYNC_LIMIT=7 
      - ZOO_LOG_LEVEL=INFO
    restart: unless-stopped
    networks:
      - kafka_networks

  kafka-0:
    image: bitnami/kafka:3.5
    container_name: kafka-server
    ports:
      - "9092:9092"
    volumes:
      - .docker/data/kafka-0:/bitnami/kafka
      - .docker/log/kafka-0:/tmp/kafka_mounts/logs
    environment:
      - KAFKA_BROKER_ID=0
      - ALLOW_PLAINTEXT_LISTENER=true
      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092
      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka-0:9092
      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
      - KAFKA_CFG_LOG_DIRS=/tmp/kafka_mounts/logs
    restart: unless-stopped
    depends_on:
      - zookeeper
    networks:
      - kafka_networks
    healthcheck:
      test: kafka-topics.sh --list --bootstrap-server kafka-server:9092 || exit -1
      interval: 5s
      timeout: 5s
      retries: 5
      start_period: 30s

  kafka-1:
    image: bitnami/kafka:3.5
    container_name: kafka-1
    ports:
      - ":9092"
    volumes:
      - .docker/data/kafka-1:/bitnami/kafka
      - .docker/log/kafka-1:/tmp/kafka_mounts/logs
    environment:
      - KAFKA_BROKER_ID=1
      - ALLOW_PLAINTEXT_LISTENER=true
      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092
      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka-1:9092
      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
      - KAFKA_CFG_LOG_DIRS=/tmp/kafka_mounts/logs
    restart: unless-stopped
    depends_on:
      - zookeeper
    networks:
      - kafka_networks
    healthcheck:
      test: kafka-topics.sh --list --bootstrap-server kafka-1:9092 || exit -1
      interval: 5s
      timeout: 5s
      retries: 5
      start_period: 30s

  kafka-2:
    image: bitnami/kafka:3.5
    container_name: kafka-2
    ports:
      - ":9092"
    volumes:
      - .docker/data/kafka-2:/bitnami/kafka
      - .docker/log/kafka-2:/tmp/kafka_mounts/logs
    environment:
      - KAFKA_BROKER_ID=2
      - ALLOW_PLAINTEXT_LISTENER=true
      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092
      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka-2:9092
      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
      - KAFKA_CFG_LOG_DIRS=/tmp/kafka_mounts/logs
    restart: unless-stopped
    depends_on:
      - zookeeper
    networks:
      - kafka_networks
    healthcheck:
      test: kafka-topics.sh --list --bootstrap-server kafka-2:9092 || exit -1
      interval: 5s
      timeout: 5s
      retries: 5
      start_period: 30s

  kafka-connect:
    build:
      context: .
      dockerfile: .docker/images/kafka-connect/Dockerfile
    container_name: kafka-connect
    ports:
      - "8083:8083"
    volumes:
      - .docker/data/kafka-connect:/data
    environment:
      - CONNECT_BOOTSTRAP_SERVERS=${CONNECT_BOOTSTRAP_SERVERS}
      - CONNECT_GROUP_ID=kafka-connectx
      - CONNECT_REPLICATION_FACTOR=3
      - CONNECT_CONFIG_STORAGE_TOPIC=_kafka-connectx-config
      - CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR=3
      - CONNECT_OFFSET_STORAGE_TOPIC=_kafka-connectx-offsets
      - CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR=3
      - CONNECT_STATUS_STORAGE_TOPIC=_kafka-connectx-status
      - CONNECT_STATUS_STORAGE_REPLICATION_FACTOR=3
      # Converters
      - CONNECT_KEY_CONVERTER=org.apache.kafka.connect.json.JsonConverter
      - CONNECT_VALUE_CONVERTER=org.apache.kafka.connect.json.JsonConverter
      - CONNECT_INTERNAL_KEY_CONVERTER=org.apache.kafka.connect.json.JsonConverter
      - CONNECT_INTERNAL_VALUE_CONVERTER=org.apache.kafka.connect.json.JsonConverter
      # Misc
      - CONNECT_REST_ADVERTISED_HOST_NAME=${CONNECT_REST_ADVERTISED_HOST_NAME}
      - CONNECT_REST_PORT=8083
      - CONNECT_LISTENERS=http://kafka-connect:8083
      - CONNECT_PLUGIN_PATH=/usr/share/java,/usr/share/confluent-hub-components
    restart: unless-stopped
    depends_on:
      - zookeeper
      - kafka-0
    networks:
      - kafka_networks

  kafka-ui:
    image: provectuslabs/kafka-ui:latest
    container_name: kafka-manager-ui
    ports:
      - "8000:8080"
    environment:
      - KAFKA_CLUSTERS_0_NAME=${KAFKA_CLUSTERS_0_NAME}
      - KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=${KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS}
      - KAFKA_CLUSTERS_0_ZOOKEEPER=${KAFKA_CLUSTERS_0_ZOOKEEPER}
      - KAFKA_CLUSTERS_0_PROPERTIES_SECURITY_PROTOCOL=${KAFKA_CLUSTERS_0_PROPERTIES_SECURITY_PROTOCOL}
      - KAFKA_CLUSTERS_0_PROPERTIES_SASL_MECHANISM=${KAFKA_CLUSTERS_0_PROPERTIES_SASL_MECHANISM}
      - KAFKA_CLUSTERS_0_PROPERTIES_SASL_JAAS_CONFIG=${KAFKA_CLUSTERS_0_PROPERTIES_SASL_JAAS_CONFIG}
      - KAFKA_CLUSTERS_0_KAFKACONNECT_0_NAME=kafka-connect
      - KAFKA_CLUSTERS_0_KAFKACONNECT_0_ADDRESS=http://kafka-connect:8083
    depends_on:
      - zookeeper
      - kafka-0
    networks:
      - kafka_networks

  sink_db:
    image: mcr.microsoft.com/mssql/server:2022-CU8-ubuntu-20.04
    container_name: sink_db
    user: root
    ports:
      - "1433:1433"
    environment:
      - ACCEPT_EULA=Y
      - MSSQL_PID=Developer
      - MSSQL_USER=SA
      - MSSQL_SA_PASSWORD=${MSSQL_SA_PASSWORD}
    volumes:
      - .docker/data/mssql:/var/opt/mssql/data
      - .docker/log/mssql:/var/opt/mssql/log
      - .docker/backups/mssql:/var/opt/mssql/backup
    restart: on-failure
    networks:
      - downstream_networks
      - kafka_networks
    healthcheck:
      test: /opt/mssql-tools/bin/sqlcmd -S localhost -U SA -P "$$MSSQL_SA_PASSWORD" -Q "SELECT 1;" || exit
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 10s

networks:
  upstream_networks:
    driver: bridge
  kafka_networks:
    driver: bridge
  downstream_networks:
    driver: bridge