Feature/split arch

.dockerignore

@@ -1,7 +1,7 @@
 venv
 __pycache__
 client
-cache
+cache*
 models
 acme.json
 audio

.gitignore

@@ -1,6 +1,6 @@
 venv
 __pycache__
-cache
+cache*
 acme.json
 models
 .env

Dockerfile

@@ -1,100 +1,20 @@
-# Builder
-FROM nvcr.io/nvidia/tensorrt:23.08-py3 as builder
-
-# Set in environment in case we need to build any extensions
-ENV TORCH_CUDA_ARCH_LIST="6.0;6.1;7.0;7.5;8.0;8.6;8.9;9.0+PTX"
-
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-        python3-dev \
-        python3-pip \
-        wget \
-        && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
-WORKDIR /root
-
-ENV ONEAPI_VERSION=2023.0.0
-RUN wget -q https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB && \
-    apt-key add *.PUB && \
-    rm *.PUB && \
-    echo "deb https://apt.repos.intel.com/oneapi all main" > /etc/apt/sources.list.d/oneAPI.list && \
-    apt-get update && \
-    apt-get install -y --no-install-recommends \
-        intel-oneapi-mkl-devel-$ONEAPI_VERSION \
-        && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
-RUN --mount=type=cache,target=/root/.cache pip install cmake==3.22.*
-
-ENV ONEDNN_VERSION=3.1.1
-RUN wget -q https://github.com/oneapi-src/oneDNN/archive/refs/tags/v${ONEDNN_VERSION}.tar.gz && \
-    tar xf *.tar.gz && \
-    rm *.tar.gz && \
-    cd oneDNN-* && \
-    cmake -DCMAKE_BUILD_TYPE=Release -DONEDNN_LIBRARY_TYPE=STATIC -DONEDNN_BUILD_EXAMPLES=OFF -DONEDNN_BUILD_TESTS=OFF -DONEDNN_ENABLE_WORKLOAD=INFERENCE -DONEDNN_ENABLE_PRIMITIVE="CONVOLUTION;REORDER" -DONEDNN_BUILD_GRAPH=OFF . && \
-    make -j$(nproc) install && \
-    cd .. && \
-    rm -r oneDNN-*
-
-RUN git clone --recursive https://github.com/OpenNMT/CTranslate2.git
-
-WORKDIR /root/CTranslate2
-
-RUN git checkout 2203ad5
-
-ARG CXX_FLAGS
-ENV CXX_FLAGS=${CXX_FLAGS:-"-msse4.1"}
-ARG CUDA_NVCC_FLAGS
-ENV CUDA_NVCC_FLAGS=${CUDA_NVCC_FLAGS:-"-Xfatbin=-compress-all"}
-ARG CUDA_ARCH_LIST
-ENV CUDA_ARCH_LIST=${TORCH_CUDA_ARCH_LIST:-"Common"}
-ENV CTRANSLATE2_ROOT=/opt/ctranslate2
-
-RUN mkdir build && \
-    cd build && \
-    cmake -DCMAKE_INSTALL_PREFIX=${CTRANSLATE2_ROOT} \
-          -DWITH_CUDA=ON -DWITH_CUDNN=ON -DWITH_MKL=ON -DWITH_DNNL=ON -DOPENMP_RUNTIME=COMP \
-          -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="${CXX_FLAGS}" \
-          -DCUDA_NVCC_FLAGS="${CUDA_NVCC_FLAGS}" -DCUDA_ARCH_LIST="${CUDA_ARCH_LIST}" .. && \
-    VERBOSE=1 make -j$(nproc) install
-
-ENV LANG=en_US.UTF-8
-COPY README.md .
-
-RUN --mount=type=cache,target=/root/.cache cd python && \
-    pip --no-cache-dir install -r install_requirements.txt && \
-    python3 setup.py bdist_wheel --dist-dir $CTRANSLATE2_ROOT
-
-# Runtime
-
 FROM nvcr.io/nvidia/tensorrt:23.08-py3
 
 WORKDIR /app
 
+# Set in environment in case we need to build any extensions
+ENV TORCH_CUDA_ARCH_LIST="6.0;6.1;6.2;7.0;7.2;7.5;8.0;8.6;8.9;9.0+PTX"
+
 # Install zstd and git-lfs for model compression and distribution
-RUN apt-get update && apt-get install -y zstd git-lfs && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y zstd  git-lfs && rm -rf /var/lib/apt/lists/*
+
+# Install our torch ver matching cuda
+RUN --mount=type=cache,target=/root/.cache pip install torch==2.2.2 torchvision==0.17.2 torchaudio==2.2.2
 
 COPY requirements.txt .
 # Run pip install with cache so we speedup subsequent rebuilds
 RUN --mount=type=cache,target=/root/.cache pip install -r requirements.txt
 
-# Install our torch ver matching cuda
-RUN --mount=type=cache,target=/root/.cache pip install -U torch==2.1.0 torchvision==0.16.0 torchaudio==2.1.0
-
-# Install compiled ctranslate2
-ENV CTRANSLATE2_ROOT=/opt/ctranslate2
-ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CTRANSLATE2_ROOT/lib
-
-COPY --from=builder $CTRANSLATE2_ROOT $CTRANSLATE2_ROOT
-RUN python3 -m pip --no-cache-dir install $CTRANSLATE2_ROOT/*.whl && \
-    rm $CTRANSLATE2_ROOT/*.whl
-
-# Install auto-gptq
-RUN --mount=type=cache,target=/root/.cache pip install auto-gptq --extra-index-url https://huggingface.github.io/autogptq-index/whl/cu118/
-
 COPY . .
 
 CMD ./entrypoint.sh

Dockerfile.nginx

@@ -0,0 +1,6 @@
+ARG NGINX_TAG
+
+FROM nginx:${NGINX_TAG}
+
+RUN apt-get update && apt-get install --no-install-recommends -y apache2-utils \
+    && rm -rf /var/lib/apt/lists/*

Dockerfile.xtts

@@ -0,0 +1,6 @@
+FROM ghcr.io/coqui-ai/xtts-streaming-server:main-cuda121-99286c10883cb9b9dcecdb6c68933c4dc0ecbec3
+WORKDIR /xtts
+
+COPY xtts/main.py .
+COPY xtts/*.json .
+EXPOSE 5002

docker-compose-cpu.yml

@@ -18,11 +18,30 @@ services:
       - ./cache:/root/.cache
     command: ./entrypoint.sh
 
+  coqui:
+    restart: unless-stopped
+    image: ${COQUI_IMAGE}:${COQUI_TAG}
+    environment:
+      - FORCE_CPU
+    env_file:
+      - .env
+    shm_size: ${SHM_SIZE}
+    ipc: host
+    ulimits:
+      memlock: -1
+      stack: 67108864
+    volumes:
+      - ./:/app
+      - ./cache:/root/.cache
+      - ./cache-local:/root/.local
+    entrypoint: /app/entrypoint-coqui.sh
+
   nginx:
     restart: unless-stopped
     depends_on:
+      - coqui
       - wis
-    image: nginx:1.25.2
+    image: ${WIS_NGINX_IMAGE}:${WIS_NGINX_TAG}
     volumes:
     - ./nginx:/nginx
     - ./nginx/nginx.conf:/etc/nginx/nginx.conf

docker-compose.yml

@@ -19,16 +19,41 @@ services:
           devices:
             - driver: nvidia
               capabilities: [gpu]
+              device_ids: ['0']
     volumes:
       - ./:/app
       - ./cache:/root/.cache
     command: ./entrypoint.sh
 
+  coqui:
+    restart: unless-stopped
+    image: ${COQUI_IMAGE}:${COQUI_TAG}
+    env_file:
+      - .env
+    shm_size: ${SHM_SIZE}
+    ipc: host
+    ulimits:
+      memlock: -1
+      stack: 67108864
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              capabilities: [gpu]
+              device_ids: ['0']
+    volumes:
+      - ./:/app
+      - ./cache:/root/.cache
+      - ./cache-local:/root/.local
+    entrypoint: /app/entrypoint-coqui.sh
+
   nginx:
     restart: unless-stopped
     depends_on:
+      - coqui
       - wis
-    image: nginx:1.25.2
+    image: ${WIS_NGINX_IMAGE}:${WIS_NGINX_TAG}
     volumes:
     - ./nginx:/nginx
     - ./nginx/nginx.conf:/etc/nginx/nginx.conf

entrypoint-coqui.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+set -e
+
+if [ "$FORCE_CPU" ]; then
+    COQUI_CUDA="false"
+else
+    COQUI_CUDA="true"
+fi
+
+export COQUI_TOS_AGREED=1
+
+if [ -r "/xtts/main.py" ]; then
+    echo "Starting coqui xtts"
+    cd /xtts
+    uvicorn main:app --host 0.0.0.0 --port 5002
+else
+    # Fix/suppress cudnn warning to not confuse people
+    ln -sf /usr/local/lib/python3.10/dist-packages/torch/lib/libnvrtc-*.so.11.2 \
+        /usr/local/lib/python3.10/dist-packages/torch/lib/libnvrtc.so
+
+    if [ "$TTS_MODEL_NAME" ]; then
+        echo "Using coqui model $TTS_MODEL_NAME"
+        python3 TTS/server/server.py --model_name "$TTS_MODEL_NAME" --use_cuda "$COQUI_CUDA"
+    else
+        echo "Using default coqui model"
+        python3 TTS/server/server.py --use_cuda "$COQUI_CUDA"
+    fi
+fi