join vllm, exllama, exllama2, split petals

mudler · Jan 6, 2024 · df373a4 · df373a4
1 parent 8a64ab8
commit df373a4
Show file tree

Hide file tree

Showing 11 changed files with 32 additions and 26 deletions.
diff --git a/backend/python/common-env/transformers/transformers-nvidia.yml b/backend/python/common-env/transformers/transformers-nvidia.yml
@@ -45,7 +45,7 @@ dependencies:
       - fsspec==2023.6.0
       - funcy==2.0
       - grpcio==1.59.0
-      - huggingface-hub==0.16.4
+      - huggingface-hub
       - idna==3.4
       - jinja2==3.1.2
       - jmespath==1.0.1
@@ -70,7 +70,6 @@ dependencies:
       - packaging==23.2
       - pandas
       - peft==0.5.0
-      - git+https://github.com/bigscience-workshop/petals
       - protobuf==4.24.4
       - psutil==5.9.5
       - pyarrow==13.0.0
@@ -85,12 +84,10 @@ dependencies:
       - scipy==1.11.3
       - six==1.16.0
       - sympy==1.12
-      - tokenizers==0.14.0
-      - torch==2.1.0
-      - torchaudio==2.1.0
+      - tokenizers
+      - torch==2.1.2
+      - torchaudio==2.1.2
       - tqdm==4.66.1
-      - transformers==4.34.0
-      - TTS==0.22.0
       - triton==2.1.0
       - typing-extensions==4.8.0
       - tzdata==2023.3
@@ -114,6 +111,7 @@ dependencies:
       - sudachipy
       - sudachidict_core
       - vocos
-      - vllm==0.2.0
-      - xformers==0.0.22
+      - vllm==0.2.7
+      - transformers>=4.36.0  # Required for Mixtral.
+      - xformers==0.0.23.post1  
 prefix: /opt/conda/envs/transformers
diff --git a/backend/python/common-env/transformers/transformers.yml b/backend/python/common-env/transformers/transformers.yml
@@ -46,7 +46,7 @@ dependencies:
       - fsspec==2023.6.0
       - funcy==2.0
       - grpcio==1.59.0
-      - huggingface-hub==0.16.4
+      - huggingface-hub
       - idna==3.4
       - jinja2==3.1.2
       - jmespath==1.0.1
@@ -59,7 +59,6 @@ dependencies:
       - packaging==23.2
       - pandas
       - peft==0.5.0
-      - git+https://github.com/bigscience-workshop/petals
       - protobuf==4.24.4
       - psutil==5.9.5
       - pyarrow==13.0.0
@@ -74,7 +73,7 @@ dependencies:
       - scipy==1.11.3
       - six==1.16.0
       - sympy==1.12
-      - tokenizers==0.14.0
+      - tokenizers
       - torch==2.1.2
       - torchaudio==2.1.2
       - tqdm==4.66.1

diff --git a/backend/python/exllama/Makefile b/backend/python/exllama/Makefile
@@ -1,8 +1,6 @@
 .PHONY: exllama
 exllama:
-	@echo "Creating virtual environment..."
-	@conda env create --name exllama --file exllama.yml
-	@echo "Virtual environment created."
+	$(MAKE) -C ../common-env/transformers
 	bash install.sh
 
 .PHONY: run

diff --git a/backend/python/exllama/install.sh b/backend/python/exllama/install.sh
@@ -5,7 +5,7 @@
 export PATH=$PATH:/opt/conda/bin
 
 # Activate conda environment
-source activate exllama
+source activate transformers
 
 echo $CONDA_PREFIX
 

diff --git a/backend/python/exllama/run.sh b/backend/python/exllama/run.sh
@@ -6,7 +6,7 @@
 export PATH=$PATH:/opt/conda/bin
 
 # Activate conda environment
-source activate exllama
+source activate transformers
 
 # get the directory where the bash script is located
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"

diff --git a/backend/python/exllama2/Makefile b/backend/python/exllama2/Makefile
@@ -1,8 +1,6 @@
 .PHONY: exllama2
 exllama2:
-	@echo "Creating virtual environment..."
-	@conda env create --name exllama2 --file exllama2.yml
-	@echo "Virtual environment created."
+	$(MAKE) -C ../common-env/transformers
 	bash install.sh
 
 .PHONY: run

diff --git a/backend/python/exllama2/install.sh b/backend/python/exllama2/install.sh
@@ -5,7 +5,7 @@
 export PATH=$PATH:/opt/conda/bin
 
 # Activate conda environment
-source activate exllama2
+source activate transformers
 
 echo $CONDA_PREFIX
 

diff --git a/backend/python/exllama2/run.sh b/backend/python/exllama2/run.sh
@@ -6,7 +6,7 @@
 export PATH=$PATH:/opt/conda/bin
 
 # Activate conda environment
-source activate exllama2
+source activate transformers
 
 # get the directory where the bash script is located
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"

diff --git a/backend/python/petals/Makefile b/backend/python/petals/Makefile
@@ -1,6 +1,8 @@
 .PHONY: petals
 petals:
-	$(MAKE) -C ../common-env/transformers
+	@echo "Creating virtual environment..."
+	@conda env create --name petals --file petals.yml
+	@echo "Virtual environment created."
 
 .PHONY: run
 run:

diff --git a/backend/python/petals/run.sh b/backend/python/petals/run.sh
@@ -5,14 +5,16 @@
 
 export PATH=$PATH:/opt/conda/bin
 
+CONDA_ENV=petals
+
 # Activate conda environment
 # if source is available use it, or use conda
 #
 if [ -f /opt/conda/bin/activate ]; then
-    source activate transformers
+    source activate $CONDA_ENV
 else
     eval "$(conda shell.bash hook)"
-    conda activate transformers
+    conda activate $CONDA_ENV
 fi
 
 # get the directory where the bash script is located

diff --git a/backend/python/petals/test.sh b/backend/python/petals/test.sh
@@ -3,7 +3,16 @@
 ## A bash script wrapper that runs the transformers server with conda
 
 # Activate conda environment
-source activate transformers
+CONDA_ENV=petals
+# Activate conda environment
+# if source is available use it, or use conda
+#
+if [ -f /opt/conda/bin/activate ]; then
+    source activate $CONDA_ENV
+else
+    eval "$(conda shell.bash hook)"
+    conda activate $CONDA_ENV
+fi
 
 # get the directory where the bash script is located
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"