diff --git a/dependencies/recommended.txt b/dependencies/recommended.txt index b9bb49ed96..05eceb7caf 100644 --- a/dependencies/recommended.txt +++ b/dependencies/recommended.txt @@ -1,23 +1,14 @@ +# Recommended because some non-commonly-used modules/examples depend on those packages. + +-f https://download.pytorch.org/whl/torch_stable.html tensorflow -torch >= 1.6+cpu, != 1.7+cpu -f https://download.pytorch.org/whl/torch_stable.html -torchvision >= 0.8+cpu -f https://download.pytorch.org/whl/torch_stable.html + +# PyTorch 1.7 has compatibility issue with model compression. +# Check for MacOS because this file is used on all platforms. +torch == 1.6.0+cpu ; sys_platform != "darwin" +torch == 1.6.0 ; sys_platform == "darwin" +torchvision == 0.7.0+cpu ; sys_platform != "darwin" +torchvision == 0.7.0 ; sys_platform == "darwin" onnx peewee -thop graphviz -tensorflow - -# the following content will be read by setup.py. -# please follow the logic in setup.py. - -# SMAC -ConfigSpaceNNI -smac4nni - -# BOHB -ConfigSpace==0.4.7 -statsmodels==0.12.0 - -# PPOTuner -enum34 -gym diff --git a/dependencies/recommended_legacy.txt b/dependencies/recommended_legacy.txt new file mode 100644 index 0000000000..96cfa447c0 --- /dev/null +++ b/dependencies/recommended_legacy.txt @@ -0,0 +1,8 @@ +-f https://download.pytorch.org/whl/torch_stable.html +tensorflow == 1.15.4 +torch == 1.5.1+cpu +torchvision == 0.6.1+cpu +keras == 2.1.6 +onnx +peewee +graphviz diff --git a/dependencies/required_extra.txt b/dependencies/required_extra.txt new file mode 100644 index 0000000000..65940c9194 --- /dev/null +++ b/dependencies/required_extra.txt @@ -0,0 +1,13 @@ +# the following content will be read by setup.py. +# please follow the logic in setup.py. + +# SMAC +ConfigSpaceNNI +smac4nni + +# BOHB +ConfigSpace==0.4.7 +statsmodels==0.12.0 + +# PPOTuner +gym diff --git a/pipelines/fast-test.yml b/pipelines/fast-test.yml index d0a7a59473..2744147c69 100644 --- a/pipelines/fast-test.yml +++ b/pipelines/fast-test.yml @@ -7,23 +7,19 @@ stages: - job: docs pool: vmImage: ubuntu-latest - variables: PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip - steps: - task: UsePythonVersion@0 inputs: versionSpec: 3.8 displayName: Configure Python version - - script: | sudo apt-get install -y pandoc sudo apt-get remove swig -y sudo apt-get install swig3.0 -y sudo ln -s /usr/bin/swig3.0 /usr/bin/swig displayName: Install apt packages - - task: Cache@2 inputs: key: 'python | "$(Agent.OS)" | dependencies/*.txt' @@ -32,15 +28,14 @@ stages: python path: $(PIP_CACHE_DIR) displayName: Cache pip packages - - script: | set -e python -m pip install -U -r dependencies/setup.txt python -m pip install -r dependencies/develop.txt python -m pip install -r dependencies/required.txt python -m pip install -r dependencies/recommended.txt + python -m pip install -r dependencies/required_extra.txt displayName: Install requirements - - script: | cd docs/en_US sphinx-build -M html . _build -W --keep-going -T @@ -49,22 +44,18 @@ stages: - job: python pool: vmImage: ubuntu-latest - variables: PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip - steps: - task: UsePythonVersion@0 inputs: versionSpec: 3.8 displayName: Configure Python version - - script: | sudo apt-get remove swig -y sudo apt-get install swig3.0 -y sudo ln -s /usr/bin/swig3.0 /usr/bin/swig displayName: Install apt packages - - task: Cache@2 inputs: key: 'python | "$(Agent.OS)" | dependencies/*.txt' @@ -73,19 +64,16 @@ stages: python path: $(PIP_CACHE_DIR) displayName: Cache pip packages - - script: | set -e python -m pip install -U -r dependencies/setup.txt python -m pip install -r dependencies/develop.txt python -m pip install -r dependencies/required.txt python -m pip install -r dependencies/recommended.txt + python -m pip install -r dependencies/required_extra.txt displayName: Install requirements - - - script: python -m pylint --rcfile pylintrc nni displayName: pylint - - script: | set -e python -m flake8 nni --count --select=E9,F63,F72,F82 --show-source --statistics @@ -96,15 +84,22 @@ stages: - job: typescript pool: vmImage: ubuntu-latest - + variables: + YARN_CACHE_FOLDER: $(Pipeline.Workspace)/.yarn steps: + - task: Cache@2 + inputs: + key: 'yarn | "$(Agent.OS)" | ts/**/yarn.lock, !**/node_modules/**' + restoreKeys: | + yarn | "$(Agent.OS)" + path: $(YARN_CACHE_FOLDER) + displayName: Cache yarn packages - script: | set -e cd ts/nni_manager yarn yarn eslint displayName: ESLint (NNI Manager) - - script: | set -e cd ts/webui @@ -118,8 +113,9 @@ stages: - job: ubuntu_latest pool: vmImage: ubuntu-latest - - # This platform tests lint and doc first. + variables: + PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip + YARN_CACHE_FOLDER: $(Pipeline.Workspace)/.yarn steps: - task: UsePythonVersion@0 @@ -127,11 +123,34 @@ stages: versionSpec: 3.8 displayName: Configure Python version + - script: | + sudo apt-get install -y pandoc + sudo apt-get remove swig -y + sudo apt-get install swig3.0 -y + sudo ln -s /usr/bin/swig3.0 /usr/bin/swig + displayName: Install apt packages + + - task: Cache@2 + inputs: + key: 'python | "$(Agent.OS)" | dependencies/*.txt' + restoreKeys: | + python | "$(Agent.OS)" + python + path: $(PIP_CACHE_DIR) + displayName: Cache pip packages + + - task: Cache@2 + inputs: + key: 'yarn | "$(Agent.OS)" | ts/**/yarn.lock, !**/node_modules/**' + restoreKeys: | + yarn | "$(Agent.OS)" + path: $(YARN_CACHE_FOLDER) + displayName: Cache yarn packages + - script: | set -e - python -m pip install --upgrade pip setuptools - python -m pip install pytest coverage - python -m pip install pylint flake8 + python -m pip install -U -r dependencies/setup.txt + python -m pip install -r dependencies/develop.txt echo "##vso[task.setvariable variable=PATH]${HOME}/.local/bin:${PATH}" displayName: Install Python tools @@ -141,41 +160,10 @@ stages: - script: | set -e - cd ts/nni_manager - yarn eslint - cd ../webui - yarn eslint - displayName: ESLint - - # FIXME: temporarily fixed to pytorch 1.6 as 1.7 won't work with compression - - - script: | - set -e - sudo apt-get install -y pandoc - python -m pip install --upgrade pygments - python -m pip install "torch==1.6.0+cpu" "torchvision==0.7.0+cpu" -f https://download.pytorch.org/whl/torch_stable.html - python -m pip install tensorflow - python -m pip install gym onnx peewee thop graphviz - python -m pip install sphinx sphinx-argparse sphinx-rtd-theme sphinxcontrib-websupport nbsphinx - sudo apt-get remove swig -y - sudo apt-get install swig3.0 -y - sudo ln -s /usr/bin/swig3.0 /usr/bin/swig - python -m pip install -e .[SMAC,BOHB] + python -m pip install -r dependencies/recommended.txt + python -m pip install -e .[SMAC,BOHB,PPOTuner] displayName: Install extra dependencies - - script: | - set -e - python -m pylint --rcfile pylintrc nni - python -m flake8 nni --count --select=E9,F63,F72,F82 --show-source --statistics - EXCLUDES=examples/trials/mnist-nas/*/mnist*.py,examples/trials/nas_cifar10/src/cifar10/general_child.py - python -m flake8 examples --count --exclude=$EXCLUDES --select=E9,F63,F72,F82 --show-source --statistics - displayName: pylint and flake8 - - - script: | - cd docs/en_US - sphinx-build -M html . _build -W --keep-going -T - displayName: Check Sphinx documentation - - script: | set -e cd test @@ -200,23 +188,48 @@ stages: python nni_test/nnitest/run_tests.py --config config/pr_tests.yml displayName: Simple integration test - - job: ubuntu_legacy pool: vmImage: ubuntu-18.04 + variables: + PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip + YARN_CACHE_FOLDER: $(Pipeline.Workspace)/.yarn # This platform runs integration test first. - steps: - task: UsePythonVersion@0 inputs: versionSpec: 3.6 displayName: Configure Python version + - script: | + sudo apt-get install -y pandoc + sudo apt-get remove swig -y + sudo apt-get install swig3.0 -y + sudo ln -s /usr/bin/swig3.0 /usr/bin/swig + displayName: Install apt packages + + - task: Cache@2 + inputs: + key: 'python | "$(Agent.OS)" | legacy | dependencies/*.txt' + restoreKeys: | + python | "$(Agent.OS)" + python + path: $(PIP_CACHE_DIR) + displayName: Cache pip packages + + - task: Cache@2 + inputs: + key: 'yarn | "$(Agent.OS)" | legacy | ts/**/yarn.lock, !**/node_modules/**' + restoreKeys: | + yarn | "$(Agent.OS)" + path: $(YARN_CACHE_FOLDER) + displayName: Cache yarn packages + - script: | set -e - python -m pip install --upgrade pip setuptools - python -m pip install pytest coverage + python -m pip install -U -r dependencies/setup.txt + python -m pip install -r dependencies/develop.txt echo "##vso[task.setvariable variable=PATH]${HOME}/.local/bin:${PATH}" displayName: Install Python tools @@ -226,12 +239,8 @@ stages: - script: | set -e - python -m pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html - python -m pip install tensorflow==1.15.4 - python -m pip install keras==2.1.6 - python -m pip install gym onnx peewee - sudo apt-get install swig -y - python -m pip install -e .[SMAC,BOHB] + python -m pip install -r dependencies/recommended_legacy.txt + python -m pip install -e .[SMAC,BOHB,PPOTuner] displayName: Install extra dependencies - script: | @@ -252,10 +261,12 @@ stages: CI=true yarn test displayName: TypeScript unit test - - job: macos pool: vmImage: macOS-10.15 + variables: + PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip + YARN_CACHE_FOLDER: $(Pipeline.Workspace)/.yarn # This platform runs TypeScript unit test first. @@ -265,11 +276,34 @@ stages: versionSpec: 3.8 displayName: Configure Python version + - script: | + brew install swig@3 + rm -f /usr/local/bin/swig + ln -s /usr/local/opt/swig\@3/bin/swig /usr/local/bin/swig + displayName: Install brew packages + + - task: Cache@2 + inputs: + key: 'python | "$(Agent.OS)" | dependencies/*.txt' + restoreKeys: | + python | "$(Agent.OS)" + python + path: $(PIP_CACHE_DIR) + displayName: Cache pip packages + + - task: Cache@2 + inputs: + key: 'yarn | "$(Agent.OS)" | ts/**/yarn.lock, !**/node_modules/**' + restoreKeys: | + yarn | "$(Agent.OS)" + path: $(YARN_CACHE_FOLDER) + displayName: Cache yarn packages + - script: | set -e + python -m pip install -U -r dependencies/setup.txt + python -m pip install -r dependencies/develop.txt echo "##vso[task.setvariable variable=PATH]${PATH}:${HOME}/.local/bin" - python -m pip install --upgrade pip setuptools wheel - python -m pip install pytest coverage displayName: Install Python tools - script: | @@ -285,13 +319,8 @@ stages: - script: | set -e - # pytorch Mac binary does not support CUDA, default is cpu version - python -m pip install torchvision==0.6.0 torch==1.5.0 - python -m pip install tensorflow==2.3.1 - brew install swig@3 - rm -f /usr/local/bin/swig - ln -s /usr/local/opt/swig\@3/bin/swig /usr/local/bin/swig - python -m pip install -e .[SMAC] + python -m pip install -r dependencies/recommended.txt + python -m pip install -e .[SMAC,BOHB,PPOTuner] displayName: Install extra dependencies - script: | @@ -304,12 +333,12 @@ stages: python nni_test/nnitest/run_tests.py --config config/pr_tests.yml displayName: Simple integration test - - # FIXME: Windows UT is still under debugging - - job: windows pool: vmImage: windows-2019 + variables: + PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip + YARN_CACHE_FOLDER: $(Pipeline.Workspace)/.yarn # This platform runs Python unit test first. @@ -319,9 +348,27 @@ stages: versionSpec: 3.8 displayName: Configure Python version + - task: Cache@2 + inputs: + key: 'python | "$(Agent.OS)" | dependencies/*.txt' + restoreKeys: | + python | "$(Agent.OS)" + python + path: $(PIP_CACHE_DIR) + displayName: Cache pip packages + + - task: Cache@2 + inputs: + key: 'yarn | "$(Agent.OS)" | ts/**/yarn.lock, !**/node_modules/**' + restoreKeys: | + yarn | "$(Agent.OS)" + path: $(YARN_CACHE_FOLDER) + displayName: Cache yarn packages + - script: | - python -m pip install --upgrade pip setuptools - python -m pip install pytest coverage + set -e + python -m pip install -U -r dependencies/setup.txt + python -m pip install -r dependencies/develop.txt displayName: Install Python tools - script: | @@ -329,9 +376,7 @@ stages: displayName: Install NNI - script: | - python -m pip install scikit-learn==0.23.2 - python -m pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html - python -m pip install tensorflow==2.3.1 + python -m pip install -r dependencies/recommended.txt displayName: Install extra dependencies - script: | diff --git a/setup.py b/setup.py index 8ed66f569d..f43b15fde0 100644 --- a/setup.py +++ b/setup.py @@ -86,9 +86,9 @@ def _setup(): python_requires = '>=3.6', install_requires = _read_requirements_txt('dependencies/required.txt'), extras_require = { - 'SMAC': _read_requirements_txt('dependencies/recommended.txt', 'SMAC'), - 'BOHB': _read_requirements_txt('dependencies/recommended.txt', 'BOHB'), - 'PPOTuner': _read_requirements_txt('dependencies/recommended.txt', 'PPOTuner') + 'SMAC': _read_requirements_txt('dependencies/required_extra.txt', 'SMAC'), + 'BOHB': _read_requirements_txt('dependencies/required_extra.txt', 'BOHB'), + 'PPOTuner': _read_requirements_txt('dependencies/required_extra.txt', 'PPOTuner') }, setup_requires = ['requests'],