docker-compose.yaml

version: "3.8"
services:
  spdk:
    image: $QUAY_SPDK:$SPDK_VERSION
    profiles:
      - build
    build:
      context: spdk/
      dockerfile: ../Dockerfile.spdk
      target: spdk
      args:
        SPDK_VERSION:
        SPDK_CEPH_VERSION:
        SPDK_PKGDEP_ARGS:
        SPDK_CONFIGURE_ARGS:
        SPDK_MAKEFLAGS:
        SPDK_NAME:
        SPDK_SUMMARY:
        SPDK_DESCRIPTION:
        SPDK_URL:
        SPDK_MAINTAINER: $MAINTAINER
        BUILD_DATE:
        SPDK_GIT_REPO:
        SPDK_GIT_BRANCH:
        SPDK_GIT_COMMIT:
      labels:
        io.ceph.nvmeof:
  # Used to export RPM packages externally (via bind mount)
  spdk-rpm-export:
    extends:
      service: spdk
    image: spdk-rpm-export
    build:
      target: rpm-export
  bdevperf:
    extends:
      service: spdk
    image: bdevperf
    build:
      target: bdevperf
    volumes:
      # sudo bash -c 'echo 2048 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages'
      # https://spdk.io/doc/containers.html
      # TODO: Pending of https://github.com/spdk/spdk/issues/2973
      - /dev/hugepages:/dev/hugepages
      - /dev/vfio/vfio:/dev/vfio/vfio
    cap_add:
      - SYS_ADMIN # huge-pages
      - CAP_SYS_NICE # RTE
      - SYS_PTRACE # gdb
    ulimits:
      nofile: $NVMEOF_NOFILE
      memlock: -1
  ceph:
    image: $QUAY_CEPH:$CEPH_CLUSTER_VERSION
    container_name: ceph
    build:
      context: .
      dockerfile: Dockerfile.ceph
      args:
        CEPH_CLUSTER_VERSION:
      labels:
        io.ceph.nvmeof:
    environment:
      CEPH_VSTART_ARGS:
      VSTART_ARGS: --without-dashboard --memstore
      TOUCHFILE: /tmp/ceph.touch
    entrypoint: >-
      sh -c './vstart.sh --new $$VSTART_ARGS &&
      ceph osd pool create rbd &&
      sleep infinity'
    healthcheck:
      test: ceph osd pool stats rbd
      start_period: 6s
      interval: 3s
    volumes:
      - ceph-conf:/etc/ceph
    networks:
      default:
        ipv4_address: 192.168.13.2
  nvmeof-base:
    build:
      context: .
      args:
        NVMEOF_TARGET: gateway
        NVMEOF_SPDK_VERSION:
        NVMEOF_NAME:
        NVMEOF_SUMMARY:
        NVMEOF_DESCRIPTION:
        NVMEOF_URL:
        NVMEOF_VERSION:
        NVMEOF_MAINTAINER: $MAINTAINER
        NVMEOF_TAGS:
        NVMEOF_WANTS:
        NVMEOF_EXPOSE_SERVICES:
        BUILD_DATE:
        NVMEOF_GIT_REPO:
        NVMEOF_GIT_BRANCH:
        NVMEOF_GIT_COMMIT:
      labels:
        io.ceph.nvmeof:
    volumes:
      # sudo bash -c 'echo 2048 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages'
      # https://spdk.io/doc/containers.html
      # TODO: Pending of https://github.com/spdk/spdk/issues/2973
      - /dev/hugepages:/dev/hugepages
      - /dev/vfio/vfio:/dev/vfio/vfio
      - $NVMEOF_CONFIG:/src/ceph-nvmeof.conf
      - /tmp/coredump:/tmp/coredump # core dump
      - ceph-conf:/etc/ceph:ro
    cap_add:
      - SYS_ADMIN # huge-pages
      - CAP_SYS_NICE # RTE
      - SYS_PTRACE # gdb
    ulimits:
      nofile: $NVMEOF_NOFILE
      memlock: -1
      core:
        soft: -1
        hard: -1
    ports:
      - "$NVMEOF_IO_PORT" # I/O controllers
      - "$NVMEOF_GW_PORT" # Gateway
      - "$NVMEOF_DISC_PORT" # Discovery
  nvmeof:
    extends:
      service: nvmeof-base
    image: $QUAY_NVMEOF:$NVMEOF_VERSION
    ports:
      - "$NVMEOF_IO_PORT" # I/O controllers
      - "$NVMEOF_GW_PORT" # Gateway
    depends_on:
      ceph:
        condition: service_healthy
  discovery:
    extends:
      service: nvmeof-base
    image: $QUAY_NVMEOF:$NVMEOF_VERSION
    ports:
      - "$NVMEOF_DISC_PORT" # Discovery
    depends_on:
      ceph:
        condition: service_healthy
    entrypoint: >-
      python3 -m control.discovery
  # Used to update lockfile (make update-lockfile)
  nvmeof-builder-base:
    extends:
      service: nvmeof-base
    image: nvmeof-builder-base
    build:
      target: builder-base
      args:
        NVMEOF_TARGET: cli
    volumes:
      - .:/src
  # Used to regenerate gRPC protobuf files
  nvmeof-builder:
    extends:
      service: nvmeof-base
    image: nvmeof-builder
    build:
      target: builder
      args:
        NVMEOF_TARGET: cli
    volumes:
      - ./control:/src/control
  # Used to export Python package
  nvmeof-python-export:
    extends:
      service: nvmeof-base
    image: nvmeof-python-export
    build:
      target: builder
      args:
        NVMEOF_TARGET: cli
  nvmeof-devel:
    image: nvmeof-devel
    # Runs from source code in current dir
    extends:
      service: nvmeof-base
    build:
      target: builder
      args:
        # https://daobook.github.io/pdm/usage/dependency/#add-development-only-dependencies
        PDM_INSTALL_DEV: "-d"
    volumes:
      - ./control:/src/control
  nvmeof-cli:
    image: $QUAY_NVMEOFCLI:$NVMEOF_VERSION
    build:
      context: .
      args:
        NVMEOF_TARGET: cli
        NVMEOF_NAME: $NVMEOF_CLI_NAME
        NVMEOF_SUMMARY: $NVMEOF_CLI_SUMMARY
        NVMEOF_DESCRIPTION: $NVMEOF_CLI_DESCRIPTION
        NVMEOF_URL:
        NVMEOF_VERSION:
        NVMEOF_MAINTAINER: $MAINTAINER
        NVMEOF_TAGS: ""
        NVMEOF_WANTS: ""
        NVMEOF_EXPOSE_SERVICES: ""
        BUILD_DATE:
        NVMEOF_GIT_REPO:
        NVMEOF_GIT_BRANCH:
        NVMEOF_GIT_COMMIT:
      labels:
        io.ceph.nvmeof:
volumes:
  ceph-conf:
networks:
  default:
    ipam:
      config:
        - subnet: 192.168.13.0/24