docker-compose-ha.yml

volumes:
  zookeeper-data:
    driver: local
  kafka-data:
    driver: local
  elasticsearch-data:
    driver: local
  kestra-data:
    driver: local

services:
  zookeeper:
    image: 'bitnami/zookeeper:latest'
    volumes:
      - zookeeper-data:/bitnami/zookeeper
    environment:
      ALLOW_ANONYMOUS_LOGIN: "yes"
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOO_LOG_LEVEL: "WARN"
    healthcheck:
      test: nc -z localhost 2181 || exit 1
      interval: 10s
      timeout: 5s
      retries: 10
      start_period: 10s

  kafka:
    image: 'bitnami/kafka:latest'
    volumes:
      - kafka-data:/bitnami
    environment:
      ALLOW_PLAINTEXT_LISTENER: "yes"
      KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
    depends_on:
      zookeeper:
        condition: service_healthy
    healthcheck:
      test: echo "exit" | curl -s -f telnet://localhost:9092 || exit 1
      interval: 30s
      timeout: 10s
      retries: 10

  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.2.3
    environment:
      discovery.type: single-node
      cluster.routing.allocation.disk.threshold_enabled: "false"
      xpack.security.enabled: "false"
      ES_JAVA_OPTS: "-Xms256m -Xmx256m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 65536
        hard: 65536
    volumes:
      - elasticsearch-data:/usr/share/elasticsearch/data
    healthcheck:
      test: curl -s -f localhost:9200/_cat/health >/dev/null || exit 1
      interval: 30s
      timeout: 10s
      retries: 10

  kestra:
    image: kestra/kestra:develop-full
    # build:
    #   context: .
    #   args:
    #     KESTRA_PLUGINS: --repositories=https://s01.oss.sonatype.org/content/repositories/snapshots io.kestra.storage:storage-gcs:LATEST io.kestra.storage:storage-minio:LATEST io.kestra.plugin:plugin-aws:LATEST io.kestra.plugin:plugin-compress:LATEST io.kestra.plugin:plugin-crypto:LATEST io.kestra.plugin:plugin-elasticsearch:LATEST io.kestra.plugin:plugin-fs:LATEST io.kestra.plugin:plugin-gcp:LATEST io.kestra.plugin:plugin-googleworkspace:LATEST io.kestra.plugin:plugin-jdbc-clickhouse:LATEST io.kestra.plugin:plugin-jdbc-mysql:LATEST io.kestra.plugin:plugin-jdbc-oracle:LATEST io.kestra.plugin:plugin-jdbc-postgres:LATEST io.kestra.plugin:plugin-jdbc-redshift:LATEST io.kestra.plugin:plugin-jdbc-sqlserver:LATEST io.kestra.plugin:plugin-jdbc-vertica:LATEST io.kestra.plugin:plugin-jdbc-vectorwise:LATEST io.kestra.plugin:plugin-kafka:LATEST io.kestra.plugin:plugin-kubernetes:LATEST io.kestra.plugin:plugin-mongodb:LATEST io.kestra.plugin:plugin-notifications:LATEST io.kestra.plugin:plugin-script-groovy:LATEST io.kestra.plugin:plugin-script-jython:LATEST io.kestra.plugin:plugin-script-nashorn:LATEST io.kestra.plugin:plugin-serdes:LATEST io.kestra.plugin:plugin-singer:LATEST
    #     APT_PACKAGES: curl zip unzip
    entrypoint: /bin/bash
    user: "root"
    command:
      - -c
      - |
        curl -s -o /tmp/examples.zip https://kestra.io/examples.zip && \
        unzip -o -d /tmp/examples /tmp/examples.zip > /dev/null && \
        /app/kestra server standalone --flow-path=/tmp/examples --worker-thread=128
    volumes:
      - kestra-data:/app/storage
      - /var/run/docker.sock:/var/run/docker.sock
      - /tmp/kestra-wd:/tmp/kestra-wd
    environment:
      KESTRA_CONFIGURATION: |
        kestra:
          kafka:
            client:
              properties:
                bootstrap.servers: kafka:9092
            defaults:
              stream:
                properties:
                  state.dir: "/tmp/kestra-wd/kafka-streams/"
          elasticsearch:
            client:
              http-hosts: http://elasticsearch:9200
          repository:
            type: elasticsearch
          storage:
            type: local
            local:
              base-path: "/app/storage"
          queue:
            type: kafka
          tasks:
            tmp-dir:
              path: /tmp/kestra-wd/tmp
          url: http://localhost:8080/
    ports:
      - "8080:8080"
    depends_on:
      kafka:
        condition: service_healthy
      zookeeper:
        condition: service_healthy
      elasticsearch:
        condition: service_healthy