From ede5dfd9f076fe5f9205bbfefe0be8f42f41418d Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Mon, 12 Sep 2022 21:17:50 +0800 Subject: [PATCH 01/30] Update configs of circleci and github action --- .circleci/config.yml | 192 +++++------------------------ .circleci/docker/Dockerfile | 15 +++ .circleci/test.yml | 234 ++++++++++++++++++++++++++++++++++++ 3 files changed, 276 insertions(+), 165 deletions(-) create mode 100644 .circleci/docker/Dockerfile create mode 100644 .circleci/test.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 4fced2cd2b..4660422fa1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,170 +1,32 @@ version: 2.1 -jobs: - lint: - docker: - - image: cimg/python:3.7.4 - steps: - - checkout - - run: - name: Install pre-commit hook - command: | - pip install pre-commit - pre-commit install - - run: - name: Linting - command: pre-commit run --all-files - build_cpu: - parameters: - # The python version must match available image tags in - # https://circleci.com/developer/images/image/cimg/python - python: - type: string - default: "3.7.0" - torch: - type: string - torchvision: - type: string - machine: - image: ubuntu-2004:202010-01 - resource_class: large - steps: - - checkout - - run: - name: Install system dependencies - command: | - sudo apt-get update - sudo apt-get install -y ffmpeg libturbojpeg ninja-build - ffmpeg -version - - run: - # https://github.com/pytorch/vision/issues/2921 - name: Install dependency of torchvision when using pyenv - command: sudo apt-get install -y liblzma-dev - - run: - # python3.7 should be re-installed due to the issue https://github.com/pytorch/vision/issues/2921 - name: Select Python - command: | - pyenv uninstall -f << parameters.python >> - pyenv install << parameters.python >> - pyenv global << parameters.python >> - - run: - name: Upgrade pip - command: | - python -m pip install pip --upgrade - - run: - name: Install PyTorch - command: python -m pip install torch==<< parameters.torch >>+cpu torchvision==<< parameters.torchvision >>+cpu -f https://download.pytorch.org/whl/torch_stable.html - - run: - name: Install MMEngine - command: pip install git+ssh://git@github.com/open-mmlab/mmengine.git - - run: - name: Install psutil - command: python -m pip install psutil - - run: - name: Build and install - command: | - rm -rf .eggs - python setup.py check -m -s - python -m pip install -e . - no_output_timeout: 20m - environment: - MMCV_WITH_OPS: 1 - - run: - name: Install dependencies of unit test - command: | - python -m pip install -r requirements/test.txt - - run: - name: Run unittests and generate coverage report - command: | - python -m coverage run --branch --source mmcv -m pytest tests/ - python -m coverage xml - python -m coverage report -m +# this allows you to use CircleCI's dynamic configuration feature +setup: true + +# the path-filtering orb is required to continue a pipeline based on +# the path of an updated fileset +orbs: + path-filtering: circleci/path-filtering@0.1.2 - build_cu102: - machine: - image: ubuntu-1604-cuda-10.1:201909-23 # the actual version of cuda is 10.2 - resource_class: gpu.nvidia.small - steps: - - checkout - - run: - name: Set CUDA environment - command: | - echo 'export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH' >> $BASH_ENV - echo 'export PATH=/usr/local/cuda/bin:$PATH' >> $BASH_ENV - echo 'export CUDA_HOME=/usr/local/cuda' >> $BASH_ENV - source $BASH_ENV - nvidia-smi - nvcc --version - gcc --version - - run: - name: Install system dependencies - command: | - sudo apt-get update - sudo apt-get install -y libturbojpeg ninja-build - # the default version of ffmpeg is 2.8.7, which should be upgraded to 4+ - sudo add-apt-repository -y ppa:jonathonf/ffmpeg-4 - sudo apt-get update - sudo apt-get install -y ffmpeg - ffmpeg -version - sudo add-apt-repository --remove ppa:jonathonf/ffmpeg-4 -y - - run: - # https://github.com/pytorch/vision/issues/2921 - name: Install dependency of torchvision when using pyenv - command: sudo apt-get install -y liblzma-dev - - run: - # python3.7 should be re-installed due to the issue https://github.com/pytorch/vision/issues/2921 - name: Select python3.7 - command: | - pyenv uninstall -f 3.7.0 - pyenv install 3.7.0 - pyenv global 3.7.0 - - run: - name: Upgrade pip - command: | - python -m pip install pip --upgrade - - run: - name: Install PyTorch - command: python -m pip install torch==1.8.1+cu102 torchvision==0.9.1+cu102 -f https://download.pytorch.org/whl/torch_stable.html - - run: - name: Install MMEngine - command: pip install git+ssh://git@github.com/open-mmlab/mmengine.git - - run: - name: Install psutil - command: python -m pip install psutil - - run: - name: Build and install - command: | - rm -rf .eggs - python setup.py check -m -s - python -m pip install -e . - environment: - MMCV_WITH_OPS: 1 - MMCV_WITH_ORT: 1 - - run: - name: Install dependencies for unit test - command: | - python -m pip install -r requirements/test.txt - - run: - name: Run unittests and generate coverage report - command: | - python -m coverage run --branch --source mmcv -m pytest tests/ - python -m coverage xml - python -m coverage report -m workflows: - unit_tests: + # the always-run workflow is always triggered, regardless of the pipeline parameters. + always-run: jobs: - - lint - - build_cpu: - name: build_py3.8_pt1.9_cpu - torch: 1.9.0 - torchvision: 0.10.0 - python: "3.8.0" - requires: - - lint - - hold: - type: approval # <<< This key-value pair will set your workflow to a status of "On Hold" - requires: - - build_py3.8_pt1.9_cpu - - build_cu102: - requires: - - hold + # the path-filtering/filter job determines which pipeline + # parameters to update. + - path-filtering/filter: + name: check-updated-files + # 3-column, whitespace-delimited mapping. One mapping per + # line: + # + mapping: | + mmcv/.* lint_only false + requirements/.* lint_only false + tests/.* lint_only false + .circleci/.* lint_only false + base-revision: main + # this is the path of the configuration we should trigger once + # path filtering and pipeline parameter value updates are + # complete. In this case, we are using the parent dynamic + # configuration itself. + config-path: .circleci/test.yml diff --git a/.circleci/docker/Dockerfile b/.circleci/docker/Dockerfile new file mode 100644 index 0000000000..7212c32e9c --- /dev/null +++ b/.circleci/docker/Dockerfile @@ -0,0 +1,15 @@ +ARG PYTORCH="1.8.1" +ARG CUDA="10.2" +ARG CUDNN="7" + +FROM pytorch/pytorch:${PYTORCH}-cuda${CUDA}-cudnn${CUDNN}-devel + +# Set MKL_THREADING_LAYER=GNU to fix issue: +# https://github.com/pytorch/pytorch/issues/37377 +ENV MKL_THREADING_LAYER GNU + +# To fix GPG key error when running apt-get update +RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub +RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub + +RUN apt-get update && apt-get install -y ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6 libgl1-mesa-glx diff --git a/.circleci/test.yml b/.circleci/test.yml new file mode 100644 index 0000000000..b49ea51ac9 --- /dev/null +++ b/.circleci/test.yml @@ -0,0 +1,234 @@ +version: 2.1 + +# the default pipeline parameters, which will be updated according to +# the results of the path-filtering orb +parameters: + lint_only: + type: boolean + default: true + +jobs: + lint: + docker: + - image: cimg/python:3.7.4 + steps: + - checkout + - run: + name: Install pre-commit hook + command: | + pip install pre-commit + pre-commit install + - run: + name: Linting + command: pre-commit run --all-files + + # build_without_torch: + # parameters: + # # The python version must match available image tags in + # # https://circleci.com/developer/images/image/cimg/python + # python: + # type: string + # default: "3.7.4" + # docker: + # - image: cimg/python:<< parameters.python >> + # resource_class: large + # steps: + # - checkout + # - run: + # name: Upgrade pip + # command: | + # pip install pip --upgrade + # pip --version + # - run: + # name: Build MMEngine from source + # command: pip install -e . -v + # - run: + # name: Install unit tests dependencies + # command: pip install -r requirements/tests.txt + # - run: + # name: Run unit tests + # command: pytest tests/test_config tests/test_registry tests/test_fileio tests/test_logging tests/test_utils --ignore=tests/test_utils/test_dl_utils + + # build_without_ops: + # parameters: + # # The python version must match available image tags in + # # https://circleci.com/developer/images/image/cimg/python + # python: + # type: string + # default: "3.7.4" + # docker: + # - image: cimg/python:<< parameters.python >> + # resource_class: large + # steps: + # - checkout + # - run: + # name: Upgrade pip + # command: | + # pip install pip --upgrade + # pip --version + # - run: + # name: Build MMEngine from source + # command: pip install -e . -v + # - run: + # name: Install unit tests dependencies + # command: pip install -r requirements/test.txt + # - run: + # name: Run unit tests + # command: pytest tests/test_config tests/test_registry tests/test_fileio tests/test_logging tests/test_utils --ignore=tests/test_utils/test_dl_utils + + build_cpu: + parameters: + # The python version must match available image tags in + # https://circleci.com/developer/images/image/cimg/python + python: + type: string + torch: + type: string + torchvision: + type: string + docker: + - image: cimg/python:<< parameters.python >> + resource_class: large + steps: + - checkout + - run: + name: Install Libraries + command: | + sudo apt-get update + sudo apt-get install -y ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6 libgl1-mesa-glx libjpeg-dev zlib1g-dev libtinfo-dev libncurses5 ffmpeg libturbojpeg + - run: + name: Configure Python & pip + command: | + pip install --upgrade pip + pip install wheel + - run: + name: Install PyTorch + command: pip install torch==<< parameters.torch >>+cpu torchvision==<< parameters.torchvision >>+cpu -f https://download.pytorch.org/whl/torch_stable.html + - run: + # pstuil is an optional package to detect the number of CPU for compiling mmcv + name: Install ninja and psutil to speed the compilation + command: pip install ninja psutil + - run: + name: Create sdist and untar + command: | + python setup.py sdist + tar zxvf dist/mmcv* -C /tmp + rm -r mmcv + - run: + name: Build and install from sdist + command: | + pushd /tmp/mmcv* + pip install -e . -v + popd + - run: + name: Install unit tests dependencies + command: pip install -r requirements/test.txt + - run: + name: Run unittests + command: | + coverage run --branch --source mmcv -m pytest tests/ + coverage xml + coverage report -m + + # build_cuda: + # parameters: + # torch: + # type: string + # cuda: + # type: enum + # enum: ["10.1", "10.2", "11.1"] + # cudnn: + # type: integer + # default: 7 + # machine: + # image: ubuntu-2004-cuda-11.4:202110-01 + # docker_layer_caching: true + # resource_class: gpu.nvidia.small + # steps: + # - checkout + # - run: + # name: Build Docker image + # command: | + # docker build .circleci/docker -t mmengine:gpu --build-arg PYTORCH=<< parameters.torch >> --build-arg CUDA=<< parameters.cuda >> --build-arg CUDNN=<< parameters.cudnn >> + # docker run --gpus all -t -d -v /home/circleci/project:/mmengine -w /mmengine --name mmengine mmengine:gpu + # - run: + # name: Build MMEngine from source + # command: | + # docker exec mmengine pip install -e . -v + # - run: + # name: Install unit tests dependencies + # command: | + # docker exec mmengine pip install -r requirements/tests.txt + # docker exec mmengine pip install openmim + # docker exec mmengine mim install 'mmcv>=2.0.0rc1' + # - run: + # name: Run unittests + # command: | + # docker exec mmengine python -m pytest tests/ + +workflows: + pr_stage_lint: + when: << pipeline.parameters.lint_only >> + jobs: + - lint: + name: lint + filters: + branches: + ignore: + - main + pr_stage_test: + when: + not: + << pipeline.parameters.lint_only >> + jobs: + - lint: + name: lint + filters: + branches: + ignore: + - main + # - build_without_torch: + # name: build without torch + # requires: + # - lint + - build_cpu: + name: minimum_version_cpu + torch: 1.6.0 + torchvision: 0.7.0 + python: 3.6.9 # The lowest python 3.6.x version available on CircleCI images + requires: + - lint + - build_cpu: + name: maximum_version_cpu + torch: 1.12.1 + torchvision: 0.13.1 + python: 3.9.0 + requires: + - minimum_version_cpu + # - hold: + # type: approval + # requires: + # - maximum_version_cpu + # - build_cuda: + # name: mainstream_version_gpu + # torch: 1.8.1 + # # Use double quotation mark to explicitly specify its type + # # as string instead of number + # cuda: "10.2" + # requires: + # - hold + # merge_stage_test: + # when: + # not: + # << pipeline.parameters.lint_only >> + # jobs: + # - build_cuda: + # name: minimum_version_gpu + # torch: 1.6.0 + # # Use double quotation mark to explicitly specify its type + # # as string instead of number + # cuda: "10.1" + # filters: + # branches: + # only: + # - main From 2d2e899d769b432d0966d0907f4df586c11e4ef7 Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Mon, 12 Sep 2022 21:21:10 +0800 Subject: [PATCH 02/30] fix base-revision --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4660422fa1..e13eabe9c6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,7 +24,7 @@ workflows: requirements/.* lint_only false tests/.* lint_only false .circleci/.* lint_only false - base-revision: main + base-revision: 2.x # this is the path of the configuration we should trigger once # path filtering and pipeline parameter value updates are # complete. In this case, we are using the parent dynamic From 27f3e5473e76a6027ddfc27aec6d138dfcf8b507 Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Mon, 12 Sep 2022 21:44:55 +0800 Subject: [PATCH 03/30] remove psutil from ci --- .circleci/test.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.circleci/test.yml b/.circleci/test.yml index b49ea51ac9..4a08239949 100644 --- a/.circleci/test.yml +++ b/.circleci/test.yml @@ -105,9 +105,8 @@ jobs: name: Install PyTorch command: pip install torch==<< parameters.torch >>+cpu torchvision==<< parameters.torchvision >>+cpu -f https://download.pytorch.org/whl/torch_stable.html - run: - # pstuil is an optional package to detect the number of CPU for compiling mmcv - name: Install ninja and psutil to speed the compilation - command: pip install ninja psutil + name: Install ninja to speed the compilation + command: pip install ninja - run: name: Create sdist and untar command: | From 1a2729de9d8a18bfd5f1a1739a5fc18fd7c4fe29 Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Mon, 12 Sep 2022 22:05:23 +0800 Subject: [PATCH 04/30] fix ci --- .circleci/docker/Dockerfile | 2 +- .circleci/test.yml | 90 +++++++++++++++++-------------------- requirements/test.txt | 4 +- 3 files changed, 45 insertions(+), 51 deletions(-) diff --git a/.circleci/docker/Dockerfile b/.circleci/docker/Dockerfile index 7212c32e9c..1cfbb3ca87 100644 --- a/.circleci/docker/Dockerfile +++ b/.circleci/docker/Dockerfile @@ -12,4 +12,4 @@ ENV MKL_THREADING_LAYER GNU RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub -RUN apt-get update && apt-get install -y ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6 libgl1-mesa-glx +RUN apt-get update && apt-get install -y ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6 libgl1-mesa-glx ffmpeg libturbojpeg diff --git a/.circleci/test.yml b/.circleci/test.yml index 4a08239949..7863e4ac3c 100644 --- a/.circleci/test.yml +++ b/.circleci/test.yml @@ -128,42 +128,36 @@ jobs: coverage run --branch --source mmcv -m pytest tests/ coverage xml coverage report -m - - # build_cuda: - # parameters: - # torch: - # type: string - # cuda: - # type: enum - # enum: ["10.1", "10.2", "11.1"] - # cudnn: - # type: integer - # default: 7 - # machine: - # image: ubuntu-2004-cuda-11.4:202110-01 - # docker_layer_caching: true - # resource_class: gpu.nvidia.small - # steps: - # - checkout - # - run: - # name: Build Docker image - # command: | - # docker build .circleci/docker -t mmengine:gpu --build-arg PYTORCH=<< parameters.torch >> --build-arg CUDA=<< parameters.cuda >> --build-arg CUDNN=<< parameters.cudnn >> - # docker run --gpus all -t -d -v /home/circleci/project:/mmengine -w /mmengine --name mmengine mmengine:gpu - # - run: - # name: Build MMEngine from source - # command: | - # docker exec mmengine pip install -e . -v - # - run: - # name: Install unit tests dependencies - # command: | - # docker exec mmengine pip install -r requirements/tests.txt - # docker exec mmengine pip install openmim - # docker exec mmengine mim install 'mmcv>=2.0.0rc1' - # - run: - # name: Run unittests - # command: | - # docker exec mmengine python -m pytest tests/ + build_cuda: + parameters: + torch: + type: string + cuda: + type: enum + enum: ["10.1", "10.2", "11.1"] + cudnn: + type: integer + default: 7 + machine: + image: ubuntu-2004-cuda-11.4:202110-01 + docker_layer_caching: true + resource_class: gpu.nvidia.small + steps: + - checkout + - run: + name: Build Docker image + command: | + docker build .circleci/docker -t mmcv:gpu --build-arg PYTORCH=<< parameters.torch >> --build-arg CUDA=<< parameters.cuda >> --build-arg CUDNN=<< parameters.cudnn >> + docker run --gpus all -t -d -v /home/circleci/project:/mmcv -w /mmcv --name mmcv mmcv:gpu + - run: + name: Build MMCV from source + command: docker exec mmcv pip install -e . -v + - run: + name: Install unit tests dependencies + command: docker exec mmcv pip install -r requirements/test.txt + - run: + name: Run unittests + command: docker exec mmcv python -m pytest tests/ workflows: pr_stage_lint: @@ -204,18 +198,18 @@ workflows: python: 3.9.0 requires: - minimum_version_cpu - # - hold: - # type: approval - # requires: - # - maximum_version_cpu - # - build_cuda: - # name: mainstream_version_gpu - # torch: 1.8.1 - # # Use double quotation mark to explicitly specify its type - # # as string instead of number - # cuda: "10.2" - # requires: - # - hold + - hold: + type: approval + requires: + - maximum_version_cpu + - build_cuda: + name: mainstream_version_gpu + torch: 1.8.1 + # Use double quotation mark to explicitly specify its type + # as string instead of number + cuda: "10.2" + requires: + - hold # merge_stage_test: # when: # not: diff --git a/requirements/test.txt b/requirements/test.txt index 9b7e0c0210..598095b7c7 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,8 +1,8 @@ coverage lmdb -onnx==1.7.0; python_version < '3.10' +onnx onnxoptimizer; python_version < '3.10' -onnxruntime>=1.8.0; python_version < '3.10' +onnxruntime>=1.8.0 protobuf~=3.19.0 pytest PyTurboJPEG From 50632947dc395fd7c391a373aecb8eb796057439 Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Mon, 12 Sep 2022 22:45:50 +0800 Subject: [PATCH 05/30] update circleci --- .circleci/test.yml | 182 ++++++++++++++++++++---------------- .github/workflows/build.yml | 1 - 2 files changed, 104 insertions(+), 79 deletions(-) diff --git a/.circleci/test.yml b/.circleci/test.yml index 7863e4ac3c..656f4622fb 100644 --- a/.circleci/test.yml +++ b/.circleci/test.yml @@ -21,61 +21,82 @@ jobs: - run: name: Linting command: pre-commit run --all-files - - # build_without_torch: - # parameters: - # # The python version must match available image tags in - # # https://circleci.com/developer/images/image/cimg/python - # python: - # type: string - # default: "3.7.4" - # docker: - # - image: cimg/python:<< parameters.python >> - # resource_class: large - # steps: - # - checkout - # - run: - # name: Upgrade pip - # command: | - # pip install pip --upgrade - # pip --version - # - run: - # name: Build MMEngine from source - # command: pip install -e . -v - # - run: - # name: Install unit tests dependencies - # command: pip install -r requirements/tests.txt - # - run: - # name: Run unit tests - # command: pytest tests/test_config tests/test_registry tests/test_fileio tests/test_logging tests/test_utils --ignore=tests/test_utils/test_dl_utils - - # build_without_ops: - # parameters: - # # The python version must match available image tags in - # # https://circleci.com/developer/images/image/cimg/python - # python: - # type: string - # default: "3.7.4" - # docker: - # - image: cimg/python:<< parameters.python >> - # resource_class: large - # steps: - # - checkout - # - run: - # name: Upgrade pip - # command: | - # pip install pip --upgrade - # pip --version - # - run: - # name: Build MMEngine from source - # command: pip install -e . -v - # - run: - # name: Install unit tests dependencies - # command: pip install -r requirements/test.txt - # - run: - # name: Run unit tests - # command: pytest tests/test_config tests/test_registry tests/test_fileio tests/test_logging tests/test_utils --ignore=tests/test_utils/test_dl_utils - + build_without_torch: + parameters: + # The python version must match available image tags in + # https://circleci.com/developer/images/image/cimg/python + python: + type: string + default: "3.7.4" + docker: + - image: cimg/python:<< parameters.python >> + resource_class: large + steps: + - checkout + - run: + name: Upgrade pip + command: | + pip install pip --upgrade + pip --version + - run: + name: Build MMCV from source + command: pip install -e . -v + environment: + MMCV_WITH_OPS: 0 + - run: + name: Install unit tests dependencies + command: pip install -r requirements/test.txt + - run: + name: Run unit tests + command: pytest tests/image tests/test_transforms tests/test_video tests/test_arraymisc.py tests/test_visualization.py tests/test_uitls/test_env.py + build_without_ops: + parameters: + # The python version must match available image tags in + # https://circleci.com/developer/images/image/cimg/python + python: + type: string + default: "3.7.4" + torch: + type: string + torchvision: + type: string + docker: + - image: cimg/python:<< parameters.python >> + resource_class: large + steps: + - checkout + - run: + name: Install Libraries + command: | + sudo apt-get update + sudo apt-get install -y ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6 libgl1-mesa-glx libjpeg-dev zlib1g-dev libtinfo-dev libncurses5 ffmpeg libturbojpeg + - run: + name: Configure Python & pip + command: | + pip install --upgrade pip + pip install wheel + - run: + name: Install PyTorch + command: pip install torch==<< parameters.torch >>+cpu torchvision==<< parameters.torchvision >>+cpu -f https://download.pytorch.org/whl/torch_stable.html + - run: + name: Create sdist and untar + command: | + sed -i "s/os.getenv('MMCV_WITH_OPS', '1')/os.getenv('MMCV_WITH_OPS', '0')/g" setup.py + python setup.py sdist + tar zxvf dist/mmcv* -C /tmp + rm -r mmcv + - run: + name: Build and install from sdist + command: | + pushd /tmp/mmcv* + pip install -e . -v + popd + - run: + name: Install unit tests dependencies + command: pip install -r requirements/test.txt + - run: + name: Run unit tests + command: pytest tests --ignore=tests/test_ops build_cpu: parameters: # The python version must match available image tags in @@ -158,7 +179,6 @@ jobs: - run: name: Run unittests command: docker exec mmcv python -m pytest tests/ - workflows: pr_stage_lint: when: << pipeline.parameters.lint_only >> @@ -168,7 +188,7 @@ workflows: filters: branches: ignore: - - main + - 2.x pr_stage_test: when: not: @@ -179,18 +199,24 @@ workflows: filters: branches: ignore: - - main - # - build_without_torch: - # name: build without torch - # requires: - # - lint + - 2.x + - build_without_torch: + name: build without torch + requires: + - lint + - build_without_ops: + name: build without ops + torch: 1.6.0 + torchvision: 0.7.0 + requires: + - build_without_torch - build_cpu: name: minimum_version_cpu torch: 1.6.0 torchvision: 0.7.0 python: 3.6.9 # The lowest python 3.6.x version available on CircleCI images requires: - - lint + - build_without_ops - build_cpu: name: maximum_version_cpu torch: 1.12.1 @@ -210,18 +236,18 @@ workflows: cuda: "10.2" requires: - hold - # merge_stage_test: - # when: - # not: - # << pipeline.parameters.lint_only >> - # jobs: - # - build_cuda: - # name: minimum_version_gpu - # torch: 1.6.0 - # # Use double quotation mark to explicitly specify its type - # # as string instead of number - # cuda: "10.1" - # filters: - # branches: - # only: - # - main + merge_stage_test: + when: + not: + << pipeline.parameters.lint_only >> + jobs: + - build_cuda: + name: minimum_version_gpu + torch: 1.6.0 + # Use double quotation mark to explicitly specify its type + # as string instead of number + cuda: "10.1" + filters: + branches: + only: + - 2.x diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3db3b343a2..bb3a108aff 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -65,7 +65,6 @@ jobs: run: | pip install -r requirements/test.txt pytest tests/ --ignore=tests/test_ops - build_cpu: runs-on: ubuntu-18.04 strategy: From 45d3a396201ca87e1bb55585a75ae169262f5d74 Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Mon, 12 Sep 2022 22:52:18 +0800 Subject: [PATCH 06/30] update circleci --- .circleci/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/test.yml b/.circleci/test.yml index 656f4622fb..7dad8245e8 100644 --- a/.circleci/test.yml +++ b/.circleci/test.yml @@ -201,11 +201,11 @@ workflows: ignore: - 2.x - build_without_torch: - name: build without torch + name: build_without_torch requires: - lint - build_without_ops: - name: build without ops + name: build_without_ops torch: 1.6.0 torchvision: 0.7.0 requires: From 993db15466a1d7ecc167a9ce4b454b3f810217cf Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Mon, 12 Sep 2022 23:00:07 +0800 Subject: [PATCH 07/30] fix typo --- .circleci/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/test.yml b/.circleci/test.yml index 7dad8245e8..0d6d2fef78 100644 --- a/.circleci/test.yml +++ b/.circleci/test.yml @@ -48,7 +48,7 @@ jobs: command: pip install -r requirements/test.txt - run: name: Run unit tests - command: pytest tests/image tests/test_transforms tests/test_video tests/test_arraymisc.py tests/test_visualization.py tests/test_uitls/test_env.py + command: pytest tests/test_image tests/test_transforms tests/test_video tests/test_arraymisc.py tests/test_visualization.py tests/test_uitls/test_env.py build_without_ops: parameters: # The python version must match available image tags in From 73698092186dad7bf90dc4c44487c9186fd5cd01 Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Mon, 12 Sep 2022 23:06:37 +0800 Subject: [PATCH 08/30] fix typo --- .circleci/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/test.yml b/.circleci/test.yml index 0d6d2fef78..d0e2982ce4 100644 --- a/.circleci/test.yml +++ b/.circleci/test.yml @@ -48,7 +48,7 @@ jobs: command: pip install -r requirements/test.txt - run: name: Run unit tests - command: pytest tests/test_image tests/test_transforms tests/test_video tests/test_arraymisc.py tests/test_visualization.py tests/test_uitls/test_env.py + command: pytest tests/test_image tests/test_transforms tests/test_video tests/test_arraymisc.py tests/test_visualization.py tests/test_utils/test_env.py build_without_ops: parameters: # The python version must match available image tags in From 7412b39e6645431dd688a19eb72fc8d61a630326 Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Mon, 12 Sep 2022 23:10:54 +0800 Subject: [PATCH 09/30] fix typo --- .circleci/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/test.yml b/.circleci/test.yml index d0e2982ce4..1d18faa789 100644 --- a/.circleci/test.yml +++ b/.circleci/test.yml @@ -48,7 +48,7 @@ jobs: command: pip install -r requirements/test.txt - run: name: Run unit tests - command: pytest tests/test_image tests/test_transforms tests/test_video tests/test_arraymisc.py tests/test_visualization.py tests/test_utils/test_env.py + command: pytest tests/test_image tests/test_transforms tests/test_video tests/test_arraymisc.py tests/test_visualization.py tests/test_utils/test_env.py --ignore=tests/test_image/test_io.py build_without_ops: parameters: # The python version must match available image tags in From 983ff99de38b4072f0a7708be89efeeb86f4d351 Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Mon, 12 Sep 2022 23:18:55 +0800 Subject: [PATCH 10/30] fix ut --- .circleci/test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.circleci/test.yml b/.circleci/test.yml index 1d18faa789..e70b40d49e 100644 --- a/.circleci/test.yml +++ b/.circleci/test.yml @@ -33,6 +33,11 @@ jobs: resource_class: large steps: - checkout + - run: + name: Install Libraries + command: | + sudo apt-get update + sudo apt-get install -y ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6 libgl1-mesa-glx libjpeg-dev zlib1g-dev libtinfo-dev libncurses5 ffmpeg libturbojpeg - run: name: Upgrade pip command: | From eedd21fa50017c7b0676583f58ddf6b06955dcad Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Tue, 13 Sep 2022 00:03:52 +0800 Subject: [PATCH 11/30] install mmengine from main branch --- .circleci/test.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.circleci/test.yml b/.circleci/test.yml index e70b40d49e..4f2e359e1a 100644 --- a/.circleci/test.yml +++ b/.circleci/test.yml @@ -83,6 +83,9 @@ jobs: - run: name: Install PyTorch command: pip install torch==<< parameters.torch >>+cpu torchvision==<< parameters.torchvision >>+cpu -f https://download.pytorch.org/whl/torch_stable.html + - run: + name: Install MMEngine from main branch + command: pip install git+ssh://git@github.com/open-mmlab/mmengine.git@main - run: name: Create sdist and untar command: | @@ -130,6 +133,9 @@ jobs: - run: name: Install PyTorch command: pip install torch==<< parameters.torch >>+cpu torchvision==<< parameters.torchvision >>+cpu -f https://download.pytorch.org/whl/torch_stable.html + - run: + name: Install MMEngine from main branch + command: pip install git+ssh://git@github.com/open-mmlab/mmengine.git@main - run: name: Install ninja to speed the compilation command: pip install ninja @@ -175,6 +181,9 @@ jobs: command: | docker build .circleci/docker -t mmcv:gpu --build-arg PYTORCH=<< parameters.torch >> --build-arg CUDA=<< parameters.cuda >> --build-arg CUDNN=<< parameters.cudnn >> docker run --gpus all -t -d -v /home/circleci/project:/mmcv -w /mmcv --name mmcv mmcv:gpu + - run: + name: Install MMEngine from main branch + command: docker exec mmcv pip install git+ssh://git@github.com/open-mmlab/mmengine.git@main - run: name: Build MMCV from source command: docker exec mmcv pip install -e . -v From c30110513f50414336dd5168560338d660661581 Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Tue, 13 Sep 2022 23:24:33 +0800 Subject: [PATCH 12/30] update github action --- .circleci/test.yml | 9 +- .github/workflows/merge_stage_test.yml | 229 +++++++++++++++++++++++++ .github/workflows/pr_stage_test.yml | 181 +++++++++++++++++++ 3 files changed, 416 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/merge_stage_test.yml create mode 100644 .github/workflows/pr_stage_test.yml diff --git a/.circleci/test.yml b/.circleci/test.yml index 4f2e359e1a..f86b54acee 100644 --- a/.circleci/test.yml +++ b/.circleci/test.yml @@ -43,6 +43,9 @@ jobs: command: | pip install pip --upgrade pip --version + - run: + name: Install MMEngine from main branch + command: pip install git+https://github.com/open-mmlab/mmengine.git@main - run: name: Build MMCV from source command: pip install -e . -v @@ -85,7 +88,7 @@ jobs: command: pip install torch==<< parameters.torch >>+cpu torchvision==<< parameters.torchvision >>+cpu -f https://download.pytorch.org/whl/torch_stable.html - run: name: Install MMEngine from main branch - command: pip install git+ssh://git@github.com/open-mmlab/mmengine.git@main + command: pip install git+https://github.com/open-mmlab/mmengine.git@main - run: name: Create sdist and untar command: | @@ -135,7 +138,7 @@ jobs: command: pip install torch==<< parameters.torch >>+cpu torchvision==<< parameters.torchvision >>+cpu -f https://download.pytorch.org/whl/torch_stable.html - run: name: Install MMEngine from main branch - command: pip install git+ssh://git@github.com/open-mmlab/mmengine.git@main + command: pip install git+https://github.com/open-mmlab/mmengine.git@main - run: name: Install ninja to speed the compilation command: pip install ninja @@ -183,7 +186,7 @@ jobs: docker run --gpus all -t -d -v /home/circleci/project:/mmcv -w /mmcv --name mmcv mmcv:gpu - run: name: Install MMEngine from main branch - command: docker exec mmcv pip install git+ssh://git@github.com/open-mmlab/mmengine.git@main + command: docker exec mmcv pip install git+https://github.com/open-mmlab/mmengine.git@main - run: name: Build MMCV from source command: docker exec mmcv pip install -e . -v diff --git a/.github/workflows/merge_stage_test.yml b/.github/workflows/merge_stage_test.yml new file mode 100644 index 0000000000..e949aa055f --- /dev/null +++ b/.github/workflows/merge_stage_test.yml @@ -0,0 +1,229 @@ +# name: merge_stage_test + +# on: +# push: +# paths-ignore: +# - ".github/**.md" +# - "docker/**" +# - "docs/**" +# - "README.md" +# - "README_zh-CN.md" +# - "CONTRIBUTING.md" +# - "CONTRIBUTING_zh-CN.md" +# - ".pre-commit-config.yaml" +# - ".pre-commit-config-zh-cn.yaml" +# branches: +# - main + +# concurrency: +# group: ${{ github.workflow }}-${{ github.ref }} +# cancel-in-progress: true + +# jobs: +# build_without_torch: +# runs-on: ubuntu-18.04 +# env: +# MMCV_WITH_OPS: 0 +# strategy: +# matrix: +# python-version: [3.7] +# steps: +# - uses: actions/checkout@v2 +# - name: Set up Python ${{ matrix.python-version }} +# uses: actions/setup-python@v2 +# with: +# python-version: ${{ matrix.python-version }} +# - name: Install system dependencies +# run: sudo apt-get update && sudo apt-get install -y ffmpeg libturbojpeg +# - name: Build MMCV from source +# run: pip install -e . -v +# - name: Install unit tests dependencies +# run: pip install -r requirements/test.txt +# - name: Run unit tests +# run: | +# pip install -r requirements/test.txt +# pytest tests/test_image tests/test_transforms tests/test_video tests/test_arraymisc.py tests/test_visualization.py tests/test_utils/test_env.py --ignore=tests/test_image/test_io.py +# build_without_ops: +# runs-on: ubuntu-18.04 +# env: +# MMCV_WITH_OPS: 0 +# strategy: +# matrix: +# python-version: [3.7] +# torch: [1.7.0, 1.8.0, 1.9.0] +# include: +# - torch: 1.7.0 +# torchvision: 0.8.1 +# - torch: 1.8.0 +# torchvision: 0.9.0 +# - torch: 1.9.0 +# torchvision: 0.10.0 +# steps: +# - uses: actions/checkout@v2 +# - name: Set up Python ${{ matrix.python-version }} +# uses: actions/setup-python@v2 +# with: +# python-version: ${{ matrix.python-version }} +# - name: Install system dependencies +# run: sudo apt-get update && sudo apt-get install -y ffmpeg libturbojpeg +# - name: Install PyTorch +# run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html +# - name: Build and install +# run: rm -rf .eggs && pip install -e . +# - name: Validate the installation +# run: python -c "import mmcv" +# - name: Run unittests +# run: | +# pip install -r requirements/test.txt +# pytest tests/ --ignore=tests/test_ops +# build_cpu_py: +# runs-on: ubuntu-18.04 +# strategy: +# matrix: +# python-version: [3.6, 3.8, 3.9] +# torch: [1.8.1] +# include: +# - torch: 1.8.1 +# torchvision: 0.9.1 +# steps: +# - uses: actions/checkout@v2 +# - name: Set up Python ${{ matrix.python-version }} +# uses: actions/setup-python@v2 +# with: +# python-version: ${{ matrix.python-version }} +# - name: Upgrade pip +# run: pip install pip --upgrade +# - name: Install PyTorch +# run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html +# - name: Build MMEngine from source +# run: pip install -e . -v +# - name: Install unit tests dependencies +# run: | +# pip install -r requirements/tests.txt +# pip install openmim +# mim install 'mmcv>=2.0.0rc1' +# - name: Run unittests and generate coverage report +# run: | +# coverage run --branch --source mmengine -m pytest tests/ +# coverage xml +# coverage report -m + +# build_cpu_pt: +# runs-on: ubuntu-18.04 +# strategy: +# matrix: +# python-version: [3.7] +# torch: [1.6.0, 1.7.1, 1.8.1, 1.9.1, 1.10.1, 1.11.0, 1.12.0] +# include: +# - torch: 1.6.0 +# torchvision: 0.7.0 +# - torch: 1.7.1 +# torchvision: 0.8.2 +# - torch: 1.8.1 +# torchvision: 0.9.1 +# - torch: 1.9.1 +# torchvision: 0.10.1 +# - torch: 1.10.1 +# torchvision: 0.11.2 +# - torch: 1.11.0 +# torchvision: 0.12.0 +# - torch: 1.12.0 +# torchvision: 0.13.0 +# steps: +# - uses: actions/checkout@v2 +# - name: Set up Python ${{ matrix.python-version }} +# uses: actions/setup-python@v2 +# with: +# python-version: ${{ matrix.python-version }} +# - name: Upgrade pip +# run: pip install pip --upgrade +# - name: Install PyTorch +# run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html +# - name: Build MMEngine from source +# run: pip install -e . -v +# - name: Install unit tests dependencies +# run: | +# pip install -r requirements/tests.txt +# pip install openmim +# mim install 'mmcv>=2.0.0rc1' +# - name: Run unittests and generate coverage report +# run: | +# coverage run --branch --source mmengine -m pytest tests/ +# coverage xml +# coverage report -m +# # Only upload coverage report for python3.7 && pytorch1.8.1 cpu +# - name: Upload coverage to Codecov +# if: ${{matrix.torch == '1.8.1' && matrix.python-version == '3.7'}} +# uses: codecov/codecov-action@v1.0.14 +# with: +# file: ./coverage.xml +# flags: unittests +# env_vars: OS,PYTHON +# name: codecov-umbrella +# fail_ci_if_error: false + +# build_cu102: +# runs-on: ubuntu-18.04 +# container: +# image: pytorch/pytorch:1.8.1-cuda10.2-cudnn7-devel +# env: +# MKL_THREADING_LAYER: GNU +# strategy: +# matrix: +# python-version: [3.7] +# include: +# - torch: 1.8.1 +# cuda: 10.2 +# steps: +# - uses: actions/checkout@v2 +# - name: Set up Python ${{ matrix.python-version }} +# uses: actions/setup-python@v2 +# with: +# python-version: ${{ matrix.python-version }} +# - name: Upgrade pip +# run: pip install pip --upgrade +# - name: Fetch GPG keys +# run: | +# apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub +# apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub +# - name: Install system dependencies +# run: apt-get update && apt-get install -y ffmpeg libsm6 libxext6 git ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6 +# - name: Build MMEngine from source +# run: pip install -e . -v +# - name: Install unit tests dependencies +# run: | +# pip install -r requirements/tests.txt +# pip install openmim +# mim install 'mmcv>=2.0.0rc1' +# - name: Run unittests and generate coverage report +# run: | +# coverage run --branch --source mmengine -m pytest tests/ +# coverage xml +# coverage report -m + +# build_windows: +# runs-on: ${{ matrix.os }} +# strategy: +# matrix: +# os: [windows-2022] +# python: [3.7] +# platform: [cpu, cu111] +# steps: +# - uses: actions/checkout@v2 +# - name: Set up Python ${{ matrix.python-version }} +# uses: actions/setup-python@v2 +# with: +# python-version: ${{ matrix.python-version }} +# - name: Upgrade pip +# run: pip install pip --upgrade +# - name: Install PyTorch +# run: pip install torch==1.8.1+${{matrix.platform}} torchvision==0.9.1+${{matrix.platform}} -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html +# - name: Build MMEngine from source +# run: pip install -e . -v +# - name: Install unit tests dependencies +# run: | +# pip install -r requirements/tests.txt +# pip install openmim +# mim install 'mmcv>=2.0.0rc1' +# - name: Run unittests +# run: pytest tests/ diff --git a/.github/workflows/pr_stage_test.yml b/.github/workflows/pr_stage_test.yml new file mode 100644 index 0000000000..92f7f338f3 --- /dev/null +++ b/.github/workflows/pr_stage_test.yml @@ -0,0 +1,181 @@ +name: pr_stage_test + +on: + pull_request: + paths-ignore: + - ".github/**.md" + - "docker/**" + - "docs/**" + - 'examples/**' + - '.dev_scripts/**' + - "README.md" + - "README_zh-CN.md" + - "CONTRIBUTING.md" + - ".pre-commit-config.yaml" + - ".pre-commit-config-zh-cn.yaml" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build_cpu: + runs-on: ubuntu-18.04 + strategy: + matrix: + python-version: [3.7] + include: + - torch: 1.8.1 + torchvision: 0.9.1 + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install system dependencies + run: sudo apt-get update && sudo apt-get install -y ffmpeg libturbojpeg + - name: Upgrade pip + run: pip install pip --upgrade + - name: Install PyTorch + run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html + - name: Install MMEngine from main branch + run: pip install git+https://github.com/open-mmlab/mmengine.git@main + - name: Install ninja to speed the compilation + run: pip install ninja + - name: Build MMCV from source + run: pip install -e . -v + - name: Install unit tests dependencies + run: pip install -r requirements/test.txt + - name: Run unittests and generate coverage report + run: | + coverage run --branch --source mmcv -m pytest tests/ + coverage xml + coverage report -m + # Upload coverage report for python3.7 && pytorch1.8.1 cpu + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v1.0.14 + with: + file: ./coverage.xml + flags: unittests + env_vars: OS,PYTHON + name: codecov-umbrella + fail_ci_if_error: false + build_cu102: + runs-on: ubuntu-18.04 + container: + image: pytorch/pytorch:1.8.1-cuda10.2-cudnn7-devel + env: + MKL_THREADING_LAYER: GNU + strategy: + matrix: + python-version: [3.7] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Upgrade pip + run: pip install pip --upgrade + - name: Fetch GPG keys + run: | + apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub + apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub + - name: Install system dependencies + run: apt-get update && apt-get install -y ffmpeg libturbojpeg + - name: Install MMEngine from main branch + run: pip install git+https://github.com/open-mmlab/mmengine.git@main + - name: Install ninja to speed the compilation + run: pip install ninja + - name: Build MMCV from source + run: pip install -e . -v + - name: Install unit tests dependencies + run: pip install -r requirements/tests.txt + - name: Run unittests and generate coverage report + run: | + coverage run --branch --source mmcv -m pytest tests/ + coverage xml + coverage report -m + build_windows_without_ops: + runs-on: ${{ matrix.os }} + env: + MMCV_WITH_OPS: 0 + strategy: + matrix: + os: [windows-2022] + python-version: [3.7] + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Upgrade pip + run: pip install pip --upgrade + - name: Install PyTorch + run: pip install torch==1.8.1+cpu torchvision==0.9.1+cpu -f https://download.pytorch.org/whl/torch_stable.html + - name: Install MMEngine from main branch + run: pip install git+https://github.com/open-mmlab/mmengine.git@main + - name: Install ninja to speed the compilation + run: pip install ninja + - name: Build MMCV from source + run: pip install -e . + - name: Install unit tests dependencies + run: pip install -r requirements/test.txt + - name: Run unittests + run: pytest tests/ --ignore tests/test_utils/test_progressbar.py --ignore tests/test_utils/test_timer.py --ignore tests/test_image/test_io.py + build_windows: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [windows-2022] + python: [3.7] + platform: [cpu, cu111] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Upgrade pip + run: pip install pip --upgrade + - name: Install PyTorch + run: pip install torch==1.8.1+${{matrix.platform}} torchvision==0.9.1+${{matrix.platform}} -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html + - name: Install MMEngine from main branch + run: pip install git+https://github.com/open-mmlab/mmengine.git@main + - name: Install ninja to speed the compilation + run: pip install ninja + - name: Build MMCV from source + run: pip install -e . + - name: Install unit tests dependencies + run: pip install -r requirements/test.txt + - name: Run unittests + run: pytest tests/ + build_macos: + runs-on: macos-latest + strategy: + matrix: + python-version: [3.7] + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install system dependencies + run: brew install ffmpeg jpeg-turbo + - name: Upgrade pip + run: pip install pip --upgrade + - name: Install PyTorch + run: pip install torch==1.8.1 torchvision==0.9.1 + - name: Install MMEngine from main branch + run: pip install git+https://github.com/open-mmlab/mmengine.git@main + - name: Install ninja to speed the compilation + run: pip install ninja + - name: Build MMCV from source + run: pip install -e . -v + - name: Install unit tests dependencies + run: pip install -r requirements/test.txt + - name: Run unittests + run: pytest tests/ From edda283439dea4aac4bbbd4a2ee1bd7b5edf24da Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Tue, 13 Sep 2022 23:48:15 +0800 Subject: [PATCH 13/30] update github action --- .github/workflows/build.yml | 120 ---------------------------- .github/workflows/pr_stage_test.yml | 2 +- 2 files changed, 1 insertion(+), 121 deletions(-) delete mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index bb3a108aff..0000000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,120 +0,0 @@ -name: build - -on: - push: - paths-ignore: - - 'README.md' - - 'README_zh-CN.md' - - 'docs/**' - - 'examples/**' - - '.dev_scripts/**' - - 'docker/**' - - pull_request: - paths-ignore: - - 'README.md' - - 'README_zh-CN.md' - - 'docs/**' - - 'examples/**' - - '.dev_scripts/**' - - 'docker/**' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MMCV_WITH_OPS: 1 - -jobs: - build_without_ops: - runs-on: ubuntu-18.04 - env: - MMCV_WITH_OPS: 0 - strategy: - matrix: - python-version: [3.7] - torch: [1.7.0, 1.8.0, 1.9.0] - include: - - torch: 1.7.0 - torchvision: 0.8.1 - - torch: 1.8.0 - torchvision: 0.9.0 - - torch: 1.9.0 - torchvision: 0.10.0 - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install system dependencies - run: sudo apt-get update && sudo apt-get install -y ffmpeg libturbojpeg openssh-client - - name: Install PyTorch - run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html - - name: Install MMEngine - run: | - eval `ssh-agent -s` - ssh-add - <<< '${{ secrets.TEST_SSH_KEY }}' - pip install git+ssh://git@github.com/open-mmlab/mmengine.git - - name: Build and install - run: rm -rf .eggs && pip install -e . - - name: Validate the installation - run: python -c "import mmcv" - - name: Run unittests - run: | - pip install -r requirements/test.txt - pytest tests/ --ignore=tests/test_ops - build_cpu: - runs-on: ubuntu-18.04 - strategy: - matrix: - python-version: [3.7] - torch: [1.5.1, 1.6.0, 1.7.0, 1.8.0, 1.9.0] - include: - - torch: 1.5.1 - torchvision: 0.6.1 - - torch: 1.6.0 - torchvision: 0.7.0 - - torch: 1.7.0 - torchvision: 0.8.1 - - torch: 1.8.0 - torchvision: 0.9.0 - - torch: 1.9.0 - torchvision: 0.10.0 - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install system dependencies - run: sudo apt-get update && sudo apt-get install -y ffmpeg libturbojpeg openssh-client - - name: Install PyTorch - run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html - - name: Install MMEngine - run: | - eval `ssh-agent -s` - ssh-add - <<< '${{ secrets.TEST_SSH_KEY }}' - pip install git+ssh://git@github.com/open-mmlab/mmengine.git - # pstuil is an optional package to detect the number of CPU for compiling mmcv - - name: Install psutil - run: pip install psutil - - name: Create sdist and untar - run: | - python setup.py sdist - tar zxvf dist/mmcv* -C /tmp - rm -r mmcv - - name: Build and install from sdist - run: | - pushd /tmp/mmcv* - pip install -e . - popd - - name: Validate the installation - run: python -c "import mmcv" - - name: Run unittests and generate coverage report - run: | - pip install -r requirements/test.txt - coverage run --branch --source=mmcv -m pytest tests/ - coverage xml - coverage report -m diff --git a/.github/workflows/pr_stage_test.yml b/.github/workflows/pr_stage_test.yml index 92f7f338f3..c49c3e76d5 100644 --- a/.github/workflows/pr_stage_test.yml +++ b/.github/workflows/pr_stage_test.yml @@ -124,7 +124,7 @@ jobs: - name: Install unit tests dependencies run: pip install -r requirements/test.txt - name: Run unittests - run: pytest tests/ --ignore tests/test_utils/test_progressbar.py --ignore tests/test_utils/test_timer.py --ignore tests/test_image/test_io.py + run: pytest tests --ignore=tests/test_ops build_windows: runs-on: ${{ matrix.os }} strategy: From 9cf9ef6c97d7cb612ac1ce69bed8550bcf9a68c9 Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Wed, 14 Sep 2022 00:12:03 +0800 Subject: [PATCH 14/30] install git --- .github/workflows/pr_stage_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr_stage_test.yml b/.github/workflows/pr_stage_test.yml index c49c3e76d5..c3a42298d6 100644 --- a/.github/workflows/pr_stage_test.yml +++ b/.github/workflows/pr_stage_test.yml @@ -83,7 +83,7 @@ jobs: apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub - name: Install system dependencies - run: apt-get update && apt-get install -y ffmpeg libturbojpeg + run: apt-get update && apt-get install -y git ffmpeg libturbojpeg - name: Install MMEngine from main branch run: pip install git+https://github.com/open-mmlab/mmengine.git@main - name: Install ninja to speed the compilation From e4b8e1ec975f1ba1d85975338e418ef181ca4a52 Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Wed, 14 Sep 2022 00:24:41 +0800 Subject: [PATCH 15/30] install git --- .github/workflows/pr_stage_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr_stage_test.yml b/.github/workflows/pr_stage_test.yml index c3a42298d6..6c681fefa7 100644 --- a/.github/workflows/pr_stage_test.yml +++ b/.github/workflows/pr_stage_test.yml @@ -91,7 +91,7 @@ jobs: - name: Build MMCV from source run: pip install -e . -v - name: Install unit tests dependencies - run: pip install -r requirements/tests.txt + run: pip install -r requirements/test.txt - name: Run unittests and generate coverage report run: | coverage run --branch --source mmcv -m pytest tests/ From 1096b3cb0bf64b5b6d42498e4b9079480f2ac2c0 Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Wed, 14 Sep 2022 00:32:15 +0800 Subject: [PATCH 16/30] update ci --- .github/workflows/pr_stage_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr_stage_test.yml b/.github/workflows/pr_stage_test.yml index 6c681fefa7..0ea07463fd 100644 --- a/.github/workflows/pr_stage_test.yml +++ b/.github/workflows/pr_stage_test.yml @@ -124,7 +124,7 @@ jobs: - name: Install unit tests dependencies run: pip install -r requirements/test.txt - name: Run unittests - run: pytest tests --ignore=tests/test_ops + run: pytest tests --ignore=tests/test_ops --ignore tests/test_image/test_io.py build_windows: runs-on: ${{ matrix.os }} strategy: @@ -151,7 +151,7 @@ jobs: - name: Install unit tests dependencies run: pip install -r requirements/test.txt - name: Run unittests - run: pytest tests/ + run: pytest tests/ --ignore tests/test_image/test_io.py build_macos: runs-on: macos-latest strategy: From 8345575e4a605af0ee9cba7a833cd152b0ac5d3c Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Wed, 14 Sep 2022 01:47:01 +0800 Subject: [PATCH 17/30] update ci of merged stage --- .github/workflows/merge_stage_test.yml | 522 ++++++++++++++----------- .github/workflows/pr_stage_test.yml | 2 - 2 files changed, 296 insertions(+), 228 deletions(-) diff --git a/.github/workflows/merge_stage_test.yml b/.github/workflows/merge_stage_test.yml index e949aa055f..da8a943302 100644 --- a/.github/workflows/merge_stage_test.yml +++ b/.github/workflows/merge_stage_test.yml @@ -1,229 +1,299 @@ -# name: merge_stage_test +name: merge_stage_test -# on: -# push: -# paths-ignore: -# - ".github/**.md" -# - "docker/**" -# - "docs/**" -# - "README.md" -# - "README_zh-CN.md" -# - "CONTRIBUTING.md" -# - "CONTRIBUTING_zh-CN.md" -# - ".pre-commit-config.yaml" -# - ".pre-commit-config-zh-cn.yaml" -# branches: -# - main +on: + push: + paths-ignore: + - ".github/**.md" + - "docker/**" + - "docs/**" + - 'examples/**' + - '.dev_scripts/**' + - "README.md" + - "README_zh-CN.md" + - "CONTRIBUTING.md" + - ".pre-commit-config.yaml" + - ".pre-commit-config-zh-cn.yaml" + branches: + - master -# concurrency: -# group: ${{ github.workflow }}-${{ github.ref }} -# cancel-in-progress: true +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true -# jobs: -# build_without_torch: -# runs-on: ubuntu-18.04 -# env: -# MMCV_WITH_OPS: 0 -# strategy: -# matrix: -# python-version: [3.7] -# steps: -# - uses: actions/checkout@v2 -# - name: Set up Python ${{ matrix.python-version }} -# uses: actions/setup-python@v2 -# with: -# python-version: ${{ matrix.python-version }} -# - name: Install system dependencies -# run: sudo apt-get update && sudo apt-get install -y ffmpeg libturbojpeg -# - name: Build MMCV from source -# run: pip install -e . -v -# - name: Install unit tests dependencies -# run: pip install -r requirements/test.txt -# - name: Run unit tests -# run: | -# pip install -r requirements/test.txt -# pytest tests/test_image tests/test_transforms tests/test_video tests/test_arraymisc.py tests/test_visualization.py tests/test_utils/test_env.py --ignore=tests/test_image/test_io.py -# build_without_ops: -# runs-on: ubuntu-18.04 -# env: -# MMCV_WITH_OPS: 0 -# strategy: -# matrix: -# python-version: [3.7] -# torch: [1.7.0, 1.8.0, 1.9.0] -# include: -# - torch: 1.7.0 -# torchvision: 0.8.1 -# - torch: 1.8.0 -# torchvision: 0.9.0 -# - torch: 1.9.0 -# torchvision: 0.10.0 -# steps: -# - uses: actions/checkout@v2 -# - name: Set up Python ${{ matrix.python-version }} -# uses: actions/setup-python@v2 -# with: -# python-version: ${{ matrix.python-version }} -# - name: Install system dependencies -# run: sudo apt-get update && sudo apt-get install -y ffmpeg libturbojpeg -# - name: Install PyTorch -# run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html -# - name: Build and install -# run: rm -rf .eggs && pip install -e . -# - name: Validate the installation -# run: python -c "import mmcv" -# - name: Run unittests -# run: | -# pip install -r requirements/test.txt -# pytest tests/ --ignore=tests/test_ops -# build_cpu_py: -# runs-on: ubuntu-18.04 -# strategy: -# matrix: -# python-version: [3.6, 3.8, 3.9] -# torch: [1.8.1] -# include: -# - torch: 1.8.1 -# torchvision: 0.9.1 -# steps: -# - uses: actions/checkout@v2 -# - name: Set up Python ${{ matrix.python-version }} -# uses: actions/setup-python@v2 -# with: -# python-version: ${{ matrix.python-version }} -# - name: Upgrade pip -# run: pip install pip --upgrade -# - name: Install PyTorch -# run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html -# - name: Build MMEngine from source -# run: pip install -e . -v -# - name: Install unit tests dependencies -# run: | -# pip install -r requirements/tests.txt -# pip install openmim -# mim install 'mmcv>=2.0.0rc1' -# - name: Run unittests and generate coverage report -# run: | -# coverage run --branch --source mmengine -m pytest tests/ -# coverage xml -# coverage report -m - -# build_cpu_pt: -# runs-on: ubuntu-18.04 -# strategy: -# matrix: -# python-version: [3.7] -# torch: [1.6.0, 1.7.1, 1.8.1, 1.9.1, 1.10.1, 1.11.0, 1.12.0] -# include: -# - torch: 1.6.0 -# torchvision: 0.7.0 -# - torch: 1.7.1 -# torchvision: 0.8.2 -# - torch: 1.8.1 -# torchvision: 0.9.1 -# - torch: 1.9.1 -# torchvision: 0.10.1 -# - torch: 1.10.1 -# torchvision: 0.11.2 -# - torch: 1.11.0 -# torchvision: 0.12.0 -# - torch: 1.12.0 -# torchvision: 0.13.0 -# steps: -# - uses: actions/checkout@v2 -# - name: Set up Python ${{ matrix.python-version }} -# uses: actions/setup-python@v2 -# with: -# python-version: ${{ matrix.python-version }} -# - name: Upgrade pip -# run: pip install pip --upgrade -# - name: Install PyTorch -# run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html -# - name: Build MMEngine from source -# run: pip install -e . -v -# - name: Install unit tests dependencies -# run: | -# pip install -r requirements/tests.txt -# pip install openmim -# mim install 'mmcv>=2.0.0rc1' -# - name: Run unittests and generate coverage report -# run: | -# coverage run --branch --source mmengine -m pytest tests/ -# coverage xml -# coverage report -m -# # Only upload coverage report for python3.7 && pytorch1.8.1 cpu -# - name: Upload coverage to Codecov -# if: ${{matrix.torch == '1.8.1' && matrix.python-version == '3.7'}} -# uses: codecov/codecov-action@v1.0.14 -# with: -# file: ./coverage.xml -# flags: unittests -# env_vars: OS,PYTHON -# name: codecov-umbrella -# fail_ci_if_error: false - -# build_cu102: -# runs-on: ubuntu-18.04 -# container: -# image: pytorch/pytorch:1.8.1-cuda10.2-cudnn7-devel -# env: -# MKL_THREADING_LAYER: GNU -# strategy: -# matrix: -# python-version: [3.7] -# include: -# - torch: 1.8.1 -# cuda: 10.2 -# steps: -# - uses: actions/checkout@v2 -# - name: Set up Python ${{ matrix.python-version }} -# uses: actions/setup-python@v2 -# with: -# python-version: ${{ matrix.python-version }} -# - name: Upgrade pip -# run: pip install pip --upgrade -# - name: Fetch GPG keys -# run: | -# apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub -# apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub -# - name: Install system dependencies -# run: apt-get update && apt-get install -y ffmpeg libsm6 libxext6 git ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6 -# - name: Build MMEngine from source -# run: pip install -e . -v -# - name: Install unit tests dependencies -# run: | -# pip install -r requirements/tests.txt -# pip install openmim -# mim install 'mmcv>=2.0.0rc1' -# - name: Run unittests and generate coverage report -# run: | -# coverage run --branch --source mmengine -m pytest tests/ -# coverage xml -# coverage report -m - -# build_windows: -# runs-on: ${{ matrix.os }} -# strategy: -# matrix: -# os: [windows-2022] -# python: [3.7] -# platform: [cpu, cu111] -# steps: -# - uses: actions/checkout@v2 -# - name: Set up Python ${{ matrix.python-version }} -# uses: actions/setup-python@v2 -# with: -# python-version: ${{ matrix.python-version }} -# - name: Upgrade pip -# run: pip install pip --upgrade -# - name: Install PyTorch -# run: pip install torch==1.8.1+${{matrix.platform}} torchvision==0.9.1+${{matrix.platform}} -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html -# - name: Build MMEngine from source -# run: pip install -e . -v -# - name: Install unit tests dependencies -# run: | -# pip install -r requirements/tests.txt -# pip install openmim -# mim install 'mmcv>=2.0.0rc1' -# - name: Run unittests -# run: pytest tests/ +jobs: + build_without_torch: + runs-on: ubuntu-18.04 + env: + MMCV_WITH_OPS: 0 + strategy: + matrix: + python-version: [3.7] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install system dependencies + run: sudo apt-get update && sudo apt-get install -y ffmpeg libturbojpeg + - name: Install MMEngine from main branch + run: pip install git+https://github.com/open-mmlab/mmengine.git@main + - name: Build MMCV from source + run: pip install -e . -v + - name: Install unit tests dependencies + run: pip install -r requirements/test.txt + - name: Run unit tests + run: pytest tests/test_image tests/test_transforms tests/test_video tests/test_arraymisc.py tests/test_visualization.py tests/test_utils/test_env.py --ignore=tests/test_image/test_io.py + build_without_ops: + runs-on: ubuntu-18.04 + env: + MMCV_WITH_OPS: 0 + strategy: + matrix: + python-version: [3.7] + torch: [1.7.0, 1.8.0, 1.9.0] + include: + - torch: 1.7.0 + torchvision: 0.8.1 + - torch: 1.8.0 + torchvision: 0.9.0 + - torch: 1.9.0 + torchvision: 0.10.0 + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install system dependencies + run: sudo apt-get update && sudo apt-get install -y ffmpeg libturbojpeg + - name: Install PyTorch + run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html + - name: Install MMEngine from main branch + run: pip install git+https://github.com/open-mmlab/mmengine.git@main + - name: Build MMCV from source + run: pip install -e . -v + - name: Install unit tests dependencies + run: pip install -r requirements/test.txt + - name: Run unittests + run: pytest tests --ignore=tests/test_ops + build_cpu_py: + runs-on: ubuntu-18.04 + strategy: + matrix: + python-version: [3.6, 3.8, 3.9] + torch: [1.8.1] + include: + - torch: 1.8.1 + torchvision: 0.9.1 + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install system dependencies + run: sudo apt-get update && sudo apt-get install -y ffmpeg libturbojpeg + - name: Upgrade pip + run: pip install pip --upgrade + - name: Install PyTorch + run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html + - name: Install MMEngine from main branch + run: pip install git+https://github.com/open-mmlab/mmengine.git@main + - name: Install ninja to speed the compilation + run: pip install ninja + - name: Build MMCV from source + run: pip install -e . -v + - name: Install unit tests dependencies + run: pip install -r requirements/test.txt + - name: Run unittests and generate coverage report + run: | + coverage run --branch --source mmcv -m pytest tests/ + coverage xml + coverage report -m + build_cpu_pt: + runs-on: ubuntu-18.04 + strategy: + matrix: + python-version: [3.7] + torch: [1.6.0, 1.7.1, 1.8.1, 1.9.1, 1.10.1, 1.11.0, 1.12.0] + include: + - torch: 1.6.0 + torchvision: 0.7.0 + - torch: 1.7.1 + torchvision: 0.8.2 + - torch: 1.8.1 + torchvision: 0.9.1 + - torch: 1.9.1 + torchvision: 0.10.1 + - torch: 1.10.1 + torchvision: 0.11.2 + - torch: 1.11.0 + torchvision: 0.12.0 + - torch: 1.12.0 + torchvision: 0.13.0 + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install system dependencies + run: sudo apt-get update && sudo apt-get install -y ffmpeg libturbojpeg + - name: Upgrade pip + run: pip install pip --upgrade + - name: Install PyTorch + run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html + - name: Install MMEngine from main branch + run: pip install git+https://github.com/open-mmlab/mmengine.git@main + - name: Install ninja to speed the compilation + run: pip install ninja + - name: Build MMCV from source + run: pip install -e . -v + - name: Install unit tests dependencies + run: pip install -r requirements/test.txt + - name: Run unittests and generate coverage report + run: | + coverage run --branch --source mmcv -m pytest tests/ + coverage xml + coverage report -m + # Only upload coverage report for python3.7 && pytorch1.8.1 cpu + - name: Upload coverage to Codecov + if: ${{matrix.torch == '1.8.1' && matrix.python-version == '3.7'}} + uses: codecov/codecov-action@v1.0.14 + with: + file: ./coverage.xml + flags: unittests + env_vars: OS,PYTHON + name: codecov-umbrella + fail_ci_if_error: false + build_cu102: + runs-on: ubuntu-18.04 + container: + image: pytorch/pytorch:1.8.1-cuda10.2-cudnn7-devel + strategy: + matrix: + python-version: [3.7] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Upgrade pip + run: pip install pip --upgrade + - name: Fetch GPG keys + run: | + apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub + apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub + - name: Install system dependencies + run: apt-get update && apt-get install -y git ffmpeg libturbojpeg + - name: Install MMEngine from main branch + run: pip install git+https://github.com/open-mmlab/mmengine.git@main + - name: Install ninja to speed the compilation + run: pip install ninja + - name: Build MMCV from source + run: pip install -e . -v + - name: Install unit tests dependencies + run: pip install -r requirements/test.txt + - name: Run unittests and generate coverage report + run: | + coverage run --branch --source mmcv -m pytest tests/ + coverage xml + coverage report -m + build_windows_without_ops: + runs-on: ${{ matrix.os }} + env: + MMCV_WITH_OPS: 0 + strategy: + matrix: + os: [windows-2022] + python-version: [3.7] + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Upgrade pip + run: pip install pip --upgrade + - name: Install PyTorch + run: pip install torch==1.8.1+cpu torchvision==0.9.1+cpu -f https://download.pytorch.org/whl/torch_stable.html + - name: Install MMEngine from main branch + run: pip install git+https://github.com/open-mmlab/mmengine.git@main + - name: Install ninja to speed the compilation + run: pip install ninja + - name: Build MMCV from source + run: pip install -e . + - name: Install unit tests dependencies + run: pip install -r requirements/test.txt + - name: Run unittests + run: pytest tests --ignore=tests/test_ops --ignore tests/test_image/test_io.py + build_windows: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [windows-2022] + python: [3.7] + torch: [1.8.1, 1.10.1] + include: + - torch: 1.8.1 + torchvision: 0.9.1 + - torch: 1.10.1 + torchvision: 0.11.2 + platform: [cpu, cu111] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Upgrade pip + run: pip install pip --upgrade + - name: Install PyTorch + run: pip install torch==1.8.1+${{matrix.platform}} torchvision==0.9.1+${{matrix.platform}} -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html + - name: Install MMEngine from main branch + run: pip install git+https://github.com/open-mmlab/mmengine.git@main + - name: Install ninja to speed the compilation + run: pip install ninja + - name: Build MMCV from source + run: pip install -e . + - name: Install unit tests dependencies + run: pip install -r requirements/test.txt + - name: Run unittests + run: pytest tests/ --ignore tests/test_image/test_io.py + build_macos: + runs-on: macos-latest + strategy: + matrix: + python-version: [3.7] + torch: [1.6.0, 1.8.1, 1.12.0] + include: + - torch: 1.6.0 + torchvision: 0.7.0 + - torch: 1.8.1 + torchvision: 0.9.1 + - torch: 1.12.0 + torchvision: 0.13.0 + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install system dependencies + run: brew install ffmpeg jpeg-turbo + - name: Upgrade pip + run: pip install pip --upgrade + - name: Install PyTorch + run: pip install torch==${{ matrix.torch }} torchvision==${{ matrix.torchvision }} + - name: Install MMEngine from main branch + run: pip install git+https://github.com/open-mmlab/mmengine.git@main + - name: Install ninja to speed the compilation + run: pip install ninja + - name: Build MMCV from source + run: pip install -e . -v + - name: Install unit tests dependencies + run: pip install -r requirements/test.txt + - name: Run unittests + run: pytest tests/ diff --git a/.github/workflows/pr_stage_test.yml b/.github/workflows/pr_stage_test.yml index 0ea07463fd..ad3c32e385 100644 --- a/.github/workflows/pr_stage_test.yml +++ b/.github/workflows/pr_stage_test.yml @@ -65,8 +65,6 @@ jobs: runs-on: ubuntu-18.04 container: image: pytorch/pytorch:1.8.1-cuda10.2-cudnn7-devel - env: - MKL_THREADING_LAYER: GNU strategy: matrix: python-version: [3.7] From d36394117bed14fdd4b59b192467472391ff530c Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Wed, 14 Sep 2022 01:50:54 +0800 Subject: [PATCH 18/30] fix typo --- .github/workflows/merge_stage_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/merge_stage_test.yml b/.github/workflows/merge_stage_test.yml index da8a943302..b7855addcc 100644 --- a/.github/workflows/merge_stage_test.yml +++ b/.github/workflows/merge_stage_test.yml @@ -14,7 +14,7 @@ on: - ".pre-commit-config.yaml" - ".pre-commit-config-zh-cn.yaml" branches: - - master + - 2.x concurrency: group: ${{ github.workflow }}-${{ github.ref }} From 61713cbc9447efda99032ca106236fc31d17948e Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Wed, 14 Sep 2022 01:55:30 +0800 Subject: [PATCH 19/30] fix typo --- .github/workflows/merge_stage_test.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/merge_stage_test.yml b/.github/workflows/merge_stage_test.yml index b7855addcc..c71b0ae726 100644 --- a/.github/workflows/merge_stage_test.yml +++ b/.github/workflows/merge_stage_test.yml @@ -13,8 +13,6 @@ on: - "CONTRIBUTING.md" - ".pre-commit-config.yaml" - ".pre-commit-config-zh-cn.yaml" - branches: - - 2.x concurrency: group: ${{ github.workflow }}-${{ github.ref }} From 6e38663579147a660b6a95465782e7bea7586f3e Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Wed, 14 Sep 2022 02:28:13 +0800 Subject: [PATCH 20/30] fix onnx compilation error --- .github/workflows/merge_stage_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/merge_stage_test.yml b/.github/workflows/merge_stage_test.yml index c71b0ae726..96eb47a404 100644 --- a/.github/workflows/merge_stage_test.yml +++ b/.github/workflows/merge_stage_test.yml @@ -91,7 +91,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install system dependencies - run: sudo apt-get update && sudo apt-get install -y ffmpeg libturbojpeg + run: sudo apt-get update && sudo apt-get install -y ffmpeg libturbojpeg protobuf-compiler - name: Upgrade pip run: pip install pip --upgrade - name: Install PyTorch From 97adbd5917f7e77af07cd119d6826f0468a09b9b Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Wed, 14 Sep 2022 02:43:24 +0800 Subject: [PATCH 21/30] fix onnx compilation error --- .github/workflows/merge_stage_test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/merge_stage_test.yml b/.github/workflows/merge_stage_test.yml index 96eb47a404..3dcfb8b24d 100644 --- a/.github/workflows/merge_stage_test.yml +++ b/.github/workflows/merge_stage_test.yml @@ -91,7 +91,9 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install system dependencies - run: sudo apt-get update && sudo apt-get install -y ffmpeg libturbojpeg protobuf-compiler + run: sudo apt-get update && sudo apt-get install -y ffmpeg libturbojpeg + - name: Install dependencies for compiling onnx if pre-built package are not available + run: pip install protobuf && apt-get -y install libprotobuf-dev protobuf-compiler cmake - name: Upgrade pip run: pip install pip --upgrade - name: Install PyTorch From f7d0732a8107fc5dadcfd43eda30efffa70f9959 Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Wed, 14 Sep 2022 10:28:58 +0800 Subject: [PATCH 22/30] use sudo to install packages --- .github/workflows/merge_stage_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/merge_stage_test.yml b/.github/workflows/merge_stage_test.yml index 3dcfb8b24d..bb3bc8e7af 100644 --- a/.github/workflows/merge_stage_test.yml +++ b/.github/workflows/merge_stage_test.yml @@ -93,7 +93,7 @@ jobs: - name: Install system dependencies run: sudo apt-get update && sudo apt-get install -y ffmpeg libturbojpeg - name: Install dependencies for compiling onnx if pre-built package are not available - run: pip install protobuf && apt-get -y install libprotobuf-dev protobuf-compiler cmake + run: pip install protobuf && sudo apt-get -y install libprotobuf-dev protobuf-compiler cmake - name: Upgrade pip run: pip install pip --upgrade - name: Install PyTorch From b0ddf2a3590d345bc931b061b9aac5e3d5f815f7 Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Wed, 14 Sep 2022 10:55:52 +0800 Subject: [PATCH 23/30] fix onnx installation --- .github/workflows/merge_stage_test.yml | 2 -- requirements/test.txt | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/merge_stage_test.yml b/.github/workflows/merge_stage_test.yml index bb3bc8e7af..c71b0ae726 100644 --- a/.github/workflows/merge_stage_test.yml +++ b/.github/workflows/merge_stage_test.yml @@ -92,8 +92,6 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system dependencies run: sudo apt-get update && sudo apt-get install -y ffmpeg libturbojpeg - - name: Install dependencies for compiling onnx if pre-built package are not available - run: pip install protobuf && sudo apt-get -y install libprotobuf-dev protobuf-compiler cmake - name: Upgrade pip run: pip install pip --upgrade - name: Install PyTorch diff --git a/requirements/test.txt b/requirements/test.txt index 598095b7c7..3de4ac6ed7 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,6 +1,7 @@ coverage lmdb -onnx +onnx<1.12.0; python_version < '3.7' +onnx>=1.12.0; python_version >= '3.7' onnxoptimizer; python_version < '3.10' onnxruntime>=1.8.0 protobuf~=3.19.0 From 606b88fce4fda0dc49969ca7b0fd40fe4cd7c861 Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Wed, 14 Sep 2022 11:37:56 +0800 Subject: [PATCH 24/30] modify python version --- .github/workflows/merge_stage_test.yml | 2 +- .github/workflows/pr_stage_test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/merge_stage_test.yml b/.github/workflows/merge_stage_test.yml index c71b0ae726..2a90b022ca 100644 --- a/.github/workflows/merge_stage_test.yml +++ b/.github/workflows/merge_stage_test.yml @@ -171,7 +171,7 @@ jobs: image: pytorch/pytorch:1.8.1-cuda10.2-cudnn7-devel strategy: matrix: - python-version: [3.7] + python-version: [3.8] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/.github/workflows/pr_stage_test.yml b/.github/workflows/pr_stage_test.yml index ad3c32e385..21143c0b8e 100644 --- a/.github/workflows/pr_stage_test.yml +++ b/.github/workflows/pr_stage_test.yml @@ -67,7 +67,7 @@ jobs: image: pytorch/pytorch:1.8.1-cuda10.2-cudnn7-devel strategy: matrix: - python-version: [3.7] + python-version: [3.8] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} From 0522b5e086220ecf80705001f10eb4a61c9170fc Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Wed, 14 Sep 2022 14:13:55 +0800 Subject: [PATCH 25/30] update --- requirements/test.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements/test.txt b/requirements/test.txt index 3de4ac6ed7..1d124d1538 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -2,7 +2,8 @@ coverage lmdb onnx<1.12.0; python_version < '3.7' onnx>=1.12.0; python_version >= '3.7' -onnxoptimizer; python_version < '3.10' +onnxoptimizer<0.3.0; python_version < '3.7' +onnxoptimizer>=0.3.0; python_version >= '3.7' onnxruntime>=1.8.0 protobuf~=3.19.0 pytest From 35ae23459978a994872e55c3e7aba4e09b2427b3 Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Wed, 14 Sep 2022 14:53:24 +0800 Subject: [PATCH 26/30] install git in circleci --- .circleci/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/test.yml b/.circleci/test.yml index f86b54acee..09589e442f 100644 --- a/.circleci/test.yml +++ b/.circleci/test.yml @@ -184,6 +184,9 @@ jobs: command: | docker build .circleci/docker -t mmcv:gpu --build-arg PYTORCH=<< parameters.torch >> --build-arg CUDA=<< parameters.cuda >> --build-arg CUDNN=<< parameters.cudnn >> docker run --gpus all -t -d -v /home/circleci/project:/mmcv -w /mmcv --name mmcv mmcv:gpu + - run: + name: Install git + command: docker exec mmcv apt-get update && apt-get install git -y - run: name: Install MMEngine from main branch command: docker exec mmcv pip install git+https://github.com/open-mmlab/mmengine.git@main From 609d8a8a791e9713f9af3945a0de46d89fa01e35 Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Wed, 14 Sep 2022 15:32:33 +0800 Subject: [PATCH 27/30] use sudo --- .circleci/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/test.yml b/.circleci/test.yml index 09589e442f..265728c4a1 100644 --- a/.circleci/test.yml +++ b/.circleci/test.yml @@ -186,7 +186,7 @@ jobs: docker run --gpus all -t -d -v /home/circleci/project:/mmcv -w /mmcv --name mmcv mmcv:gpu - run: name: Install git - command: docker exec mmcv apt-get update && apt-get install git -y + command: docker exec mmcv sudo apt-get update && sudo apt-get install git -y - run: name: Install MMEngine from main branch command: docker exec mmcv pip install git+https://github.com/open-mmlab/mmengine.git@main From a2e81321a94478f645eccc2f9ef383406feb9b27 Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Wed, 14 Sep 2022 17:04:36 +0800 Subject: [PATCH 28/30] install git --- .circleci/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/docker/Dockerfile b/.circleci/docker/Dockerfile index 1cfbb3ca87..db5ab86e3a 100644 --- a/.circleci/docker/Dockerfile +++ b/.circleci/docker/Dockerfile @@ -12,4 +12,4 @@ ENV MKL_THREADING_LAYER GNU RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub -RUN apt-get update && apt-get install -y ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6 libgl1-mesa-glx ffmpeg libturbojpeg +RUN apt-get update && apt-get install -y ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6 libgl1-mesa-glx ffmpeg libturbojpeg git From 422b3c6c4f5ddf1e9575da89038c5ba60ef226a4 Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Wed, 14 Sep 2022 17:08:52 +0800 Subject: [PATCH 29/30] update --- .circleci/test.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.circleci/test.yml b/.circleci/test.yml index 265728c4a1..f86b54acee 100644 --- a/.circleci/test.yml +++ b/.circleci/test.yml @@ -184,9 +184,6 @@ jobs: command: | docker build .circleci/docker -t mmcv:gpu --build-arg PYTORCH=<< parameters.torch >> --build-arg CUDA=<< parameters.cuda >> --build-arg CUDNN=<< parameters.cudnn >> docker run --gpus all -t -d -v /home/circleci/project:/mmcv -w /mmcv --name mmcv mmcv:gpu - - run: - name: Install git - command: docker exec mmcv sudo apt-get update && sudo apt-get install git -y - run: name: Install MMEngine from main branch command: docker exec mmcv pip install git+https://github.com/open-mmlab/mmengine.git@main From 6df1210ea93609cd7b18ff38e513fc925bf6c412 Mon Sep 17 00:00:00 2001 From: zhouzaida Date: Wed, 14 Sep 2022 19:01:49 +0800 Subject: [PATCH 30/30] test on 2.x branch --- .github/workflows/merge_stage_test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/merge_stage_test.yml b/.github/workflows/merge_stage_test.yml index 2a90b022ca..a43919ea9e 100644 --- a/.github/workflows/merge_stage_test.yml +++ b/.github/workflows/merge_stage_test.yml @@ -13,6 +13,8 @@ on: - "CONTRIBUTING.md" - ".pre-commit-config.yaml" - ".pre-commit-config-zh-cn.yaml" + branches: + - 2.x concurrency: group: ${{ github.workflow }}-${{ github.ref }}