Reduce dependency install time and decrease docker image size

docker/Dockerfile.dev

@@ -65,8 +65,10 @@ RUN if [ "$MACHINE_TYPE" = "gpu" ]; then export USE_CUDA=1; fi \
     && git clone https://github.com/pytorch/serve.git \
     && cd serve \
     && git checkout ${BRANCH_NAME} \
-    && python$PYTHON_VERSION -m venv /home/venv \
-    && python -m pip install -U pip setuptools \
+    && python$PYTHON_VERSION -m venv /home/venv
+ENV PATH="/home/venv/bin:$PATH"
+WORKDIR serve
+RUN python -m pip install -U pip setuptools \
     && if [ -z "$CUDA_VERSION" ]; then python ts_scripts/install_dependencies.py --environment=dev; else python ts_scripts/install_dependencies.py --environment=dev  --cuda $CUDA_VERSION; fi \
     && if [ "$BUILD_WITH_IPEX" = "true" ]; then python -m pip install --no-cache-dir intel_extension_for_pytorch==${IPEX_VERSION} -f ${IPEX_URL}; fi \
     && python ts_scripts/install_from_src.py \

requirements/common.txt

@@ -1,11 +1,6 @@
-psutil
-cython
-wheel
-requests
-pillow==9.3.0
-captum
-packaging
-numpy; sys_platform != 'win32'
-numpy==1.19.3; sys_platform == 'win32' #see https://tinyurl.com/y3dm3h86
+psutil==5.9.5
+requests==2.31.0
+captum==0.6.0
+packaging==23.1
 pynvml==11.4.1
-pyyaml
+pyyaml==6.0

requirements/developer.txt

@@ -1,18 +1,17 @@
 -r common.txt
-mock
-pytest
+mock==5.0.2
+pytest==7.3.1
 pylint==2.6.0
-pytest-mock
-pytest-cov
-grpcio
-protobuf
-grpcio-tools
+pytest-mock==3.10.0
+pytest-cov==4.1.0
+grpcio==1.54.2
+protobuf==4.23.1
+grpcio-tools==1.54.2
 transformers==4.28.1
-pyspelling
-pygit2
-pyspelling
-pre-commit
-twine
-mypy
-torchpippy
-intel_extension_for_pytorch; sys_platform != 'win32' and sys_platform != 'darwin'
+pyspelling==2.8.2
+pygit2==1.12.1
+pre-commit==3.3.2
+twine==4.0.2
+mypy==1.3.0
+torchpippy==0.1.1
+intel_extension_for_pytorch==2.0.100; sys_platform != 'win32' and sys_platform != 'darwin'

requirements/torch_common.txt

@@ -0,0 +1,5 @@
+cython==0.29.34
+wheel==0.40.0
+pillow==9.3.0
+numpy==1.24.3; sys_platform != 'win32'
+numpy==1.19.3; sys_platform == 'win32' #see https://tinyurl.com/y3dm3h86

requirements/torch_cu101_linux.txt

@@ -1,8 +1,6 @@
 #pip install torch==1.8.1+cu101 torchvision==0.9.1+cu101 torchaudio==0.8.1 -f https://download.pytorch.org/whl/torch_stable.html
 --find-links https://download.pytorch.org/whl/torch_stable.html
-cython
-wheel
-pillow==9.3.0
+-r torch_common.txt
 torch==1.8.1+cu101; sys_platform == 'linux'
 torchvision==0.9.1+cu101; sys_platform == 'linux'
 torchtext==0.9.1; sys_platform == 'linux'

requirements/torch_cu101_windows.txt

@@ -1,6 +1,7 @@
 #pip install torch==1.8.1+cu101 torchvision==0.9.1+cu101 torchaudio===0.8.1 -f https://download.pytorch.org/whl/torch_stable.html
 --find-links https://download.pytorch.org/whl/torch_stable.html
+-r torch_common.txt
 torch==1.8.1+cu101; sys_platform == 'win32'
 torchvision==0.9.1+cu101; sys_platform == 'win32'
 torchtext==0.9.1; sys_platform == 'win32'
-torchaudio==0.8.1; sys_platform == 'win32'
+torchaudio==0.8.1; sys_platform == 'win32'

requirements/torch_cu102_linux.txt

@@ -1,7 +1,5 @@
 # pip install torch torchvision torchaudio
-cython
-wheel
-pillow==9.3.0
+-r torch_common.txt
 torch==1.12.0; sys_platform == 'linux'
 torchvision==0.13.0; sys_platform == 'linux'
 torchtext==0.13.0; sys_platform == 'linux'

requirements/torch_cu102_windows.txt

@@ -1,6 +1,7 @@
 #pip install torch===1.9.0 torchvision===0.10.0 torchaudio===0.9.0 -f https://download.pytorch.org/whl/torch_stable.html
 --find-links https://download.pytorch.org/whl/torch_stable.html
+-r torch_common.txt
 torch==1.10.0+cu102; sys_platform == 'win32'
 torchvision==0.11.1+cu102; sys_platform == 'win32'
 torchtext==0.11.0; sys_platform == 'win32'
-torchaudio==0.10.0+cu102; sys_platform == 'win32'
+torchaudio==0.10.0+cu102; sys_platform == 'win32'

requirements/torch_cu111_linux.txt

@@ -1,8 +1,6 @@
 #pip install torch==1.9.0+cu111 torchvision==0.10.0+cu111 torchaudio===0.9.0 -f https://download.pytorch.org/whl/torch_stable.html
 --find-links https://download.pytorch.org/whl/torch_stable.html
-cython
-wheel
-pillow==9.3.0
+-r torch_common.txt
 torch==1.9.0+cu111; sys_platform == 'linux'
 torchvision==0.10.0+cu111; sys_platform == 'linux'
 torchtext==0.10.0; sys_platform == 'linux'

requirements/torch_cu111_windows.txt

@@ -1,5 +1,6 @@
 #pip install torch===1.9.0+cu111 torchvision===0.10.0+cu111 torchaudio===0.9.0 -f https://download.pytorch.org/whl/torch_stable.html
 --find-links https://download.pytorch.org/whl/torch_stable.html
+-r torch_common.txt
 torch==1.9.0+cu111; sys_platform == 'win32'
 torchvision==0.10.0+cu111; sys_platform == 'win32'
 torchtext==0.10.0; sys_platform == 'win32'

requirements/torch_cu113_linux.txt

@@ -1,8 +1,6 @@
 #pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu113
 --extra-index-url https://download.pytorch.org/whl/cu113
-cython
-wheel
-pillow==9.3.0
+-r torch_common.txt
 torch==1.12.0+cu113; sys_platform == 'linux'
 torchvision==0.13.0+cu113; sys_platform == 'linux'
 torchtext==0.13.0; sys_platform == 'linux'

requirements/torch_cu113_windows.txt

@@ -1,5 +1,6 @@
 #pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu113
 --extra-index-url https://download.pytorch.org/whl/cu113
+-r torch_common.txt
 torch==1.12.0+cu113; sys_platform == 'win32'
 torchvision==0.13.0+cu113; sys_platform == 'win32'
 torchtext==0.13.0; sys_platform == 'win32'

requirements/torch_cu116_linux.txt

@@ -1,8 +1,6 @@
 #pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu116
 --extra-index-url https://download.pytorch.org/whl/cu116
-cython
-wheel
-pillow==9.3.0
+-r torch_common.txt
 torch==1.13.1+cu116; sys_platform == 'linux'
 torchvision==0.14.1+cu116; sys_platform == 'linux'
 torchtext==0.14.1; sys_platform == 'linux'

requirements/torch_cu116_windows.txt

@@ -1,5 +1,6 @@
 #pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu116
 --extra-index-url https://download.pytorch.org/whl/cu116
+-r torch_common.txt
 torch==1.13.1+cu116; sys_platform == 'win32'
 torchvision==0.14.1+cu116; sys_platform == 'win32'
 torchtext==0.14.1; sys_platform == 'win32'

requirements/torch_cu117_linux.txt

@@ -1,5 +1,6 @@
 #pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu117
 --extra-index-url https://download.pytorch.org/whl/cu117
+-r torch_common.txt
 torch==2.0.0+cu117; sys_platform == 'linux'
 torchvision==0.15.1+cu117; sys_platform == 'linux'
 torchtext==0.15.1; sys_platform == 'linux'

requirements/torch_cu117_windows.txt

@@ -1,5 +1,6 @@
 #pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu117
 --extra-index-url https://download.pytorch.org/whl/cu117
+-r torch_common.txt
 torch==2.0.0+cu117; sys_platform == 'win32'
 torchvision==0.15.1+cu117; sys_platform == 'win32'
 torchtext==0.15.1; sys_platform == 'win32'

requirements/torch_cu118_linux.txt

@@ -1,5 +1,6 @@
 #pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu117
 --extra-index-url https://download.pytorch.org/whl/cu118
+-r torch_common.txt
 torch==2.0.0+cu118; sys_platform == 'linux'
 torchvision==0.15.1+cu118; sys_platform == 'linux'
 torchtext==0.15.1; sys_platform == 'linux'

requirements/torch_cu118_windows.txt

@@ -1,5 +1,6 @@
 #pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu117
 --extra-index-url https://download.pytorch.org/whl/cu118
+-r torch_common.txt
 torch==2.0.0+cu118; sys_platform == 'linux'
 torchvision==0.15.1+cu118; sys_platform == 'linux'
 torchtext==0.15.1; sys_platform == 'linux'

requirements/torch_cu92.txt

@@ -1,4 +1,5 @@
+-r torch_common.txt
 torch==1.8.1+cu92
 torchvision==0.9.1+cu92
 torchtext==0.9.1
-torchaudio==0.8.1
+torchaudio==0.8.1

requirements/torch_cu92_linux.txt

@@ -1,8 +1,6 @@
 #pip install torch==1.8.1+cu92 torchvision==0.9.1+cu92 torchaudio==0.8.1 -f https://download.pytorch.org/whl/torch_stable.html
 --find-links https://download.pytorch.org/whl/torch_stable.html
-cython
-wheel
-pillow==9.3.0
+-r torch_common.txt
 torch==1.8.1+cu92; sys_platform == 'linux'
 torchvision==0.9.1+cu92; sys_platform == 'linux'
 torchtext==0.9.1; sys_platform == 'linux'

requirements/torch_darwin.txt

@@ -1,4 +1,5 @@
 #pip install torch torchvision torchaudio
+-r torch_common.txt
 torch==2.0.0; sys_platform == 'darwin'
 torchvision==0.15.1; sys_platform == 'darwin'
 torchtext==0.15.1; sys_platform == 'darwin'

requirements/torch_linux.txt

@@ -1,5 +1,6 @@
 #pip3 install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu
 --extra-index-url https://download.pytorch.org/whl/cpu
+-r torch_common.txt
 torch==2.0.0+cpu; sys_platform == 'linux'
 torchvision==0.15.1+cpu; sys_platform == 'linux'
 torchtext==0.15.1; sys_platform == 'linux'

requirements/torch_windows.txt

@@ -1,4 +1,5 @@
 #pip install torch torchvision torchaudio
+-r torch_common.txt
 torch==2.0.0; sys_platform == 'win32'
 torchvision==0.15.1; sys_platform == 'win32'
 torchtext==0.15.1; sys_platform == 'win32'

ts_scripts/install_dependencies.py

@@ -50,6 +50,14 @@ def install_python_packages(self, cuda_version, requirements_file_path, nightly)
             # as it may reinstall the packages with different versions
             os.system("conda install -y conda-build")
 
+        # Install PyTorch packages
+        if nightly:
+            os.system(
+                f"pip3 install numpy --pre torch torchvision torchtext torchaudio --force-reinstall --extra-index-url https://download.pytorch.org/whl/nightly/{cuda_version}"
+            )
+        else:
+            self.install_torch_packages(cuda_version)
+
         os.system(f"{sys.executable} -m pip install -U pip setuptools")
         # developer.txt also installs packages from common.txt
         os.system(f"{sys.executable} -m pip install -U -r {requirements_file_path}")
@@ -59,14 +67,6 @@ def install_python_packages(self, cuda_version, requirements_file_path, nightly)
             gpu_requirements_file = os.path.join("requirements", "common_gpu.txt")
             os.system(f"{sys.executable} -m pip install -U -r {gpu_requirements_file}")
 
-        # Install PyTorch packages
-        if nightly:
-            os.system(
-                f"pip3 install numpy --pre torch torchvision torchtext torchaudio --force-reinstall --extra-index-url https://download.pytorch.org/whl/nightly/{cuda_version}"
-            )
-        else:
-            self.install_torch_packages(cuda_version)
-
     def install_node_packages(self):
         os.system(
             f"{self.sudo_cmd}npm install -g newman newman-reporter-htmlextra markdown-link-check"
@@ -106,14 +106,6 @@ def install_wget(self):
         if os.system("wget --version") != 0 or args.force:
             os.system(f"{self.sudo_cmd}apt-get install -y wget")
 
-    def install_libgit2(self):
-        os.system(
-            f"wget https://github.com/libgit2/libgit2/archive/refs/tags/v1.3.0.tar.gz -O libgit2-1.3.0.tar.gz"
-        )
-        os.system(f"tar xzf libgit2-1.3.0.tar.gz")
-        os.system(f"cd libgit2-1.3.0 && cmake . && make && sudo make install && cd ..")
-        os.system(f"rm -rf libgit2-1.3.0 && rm libgit2-1.3.0.tar.gz")
-
     def install_numactl(self):
         if os.system("numactl --show") != 0 or args.force:
             os.system(f"{self.sudo_cmd}apt-get install -y numactl")
@@ -174,9 +166,6 @@ def install_dependencies(cuda_version=None, nightly=False):
         system.install_nodejs()
         system.install_node_packages()
 
-    if platform.system() == "Linux" and args.environment == "dev":
-        system.install_libgit2()
-
     # Sequence of installation to be maintained
     system.install_java()