Update circleci (pytorch#2306)

Summary:
Stack of PRs for code sync
Pull Request resolved: pytorch#2306

Reviewed By: lw

Differential Revision: D21971343

Pulled By: fmassa

fbshipit-source-id: bda18d47672729bb7aff8891c17ebdf9ec076475

.circleci/config.yml.in

@@ -6,14 +6,17 @@ version: 2.1
 #     - Replace binary_linux_wheel_py3.7 with the name of the job you want to test.
 #       Job names are 'name:' key.
 
-orbs:
-  win: circleci/windows@2.0.0
-
 executors:
-  windows-gpu-prototype:
+  windows-cpu:
     machine:
-      resource_class: windows.gpu.small.prototype
-      image: windows-server-2019-nvidia:201908-28
+      resource_class: windows.xlarge
+      image: windows-server-2019-vs2019:stable
+      shell: bash.exe
+
+  windows-gpu:
+    machine:
+      resource_class: windows.gpu.nvidia.medium
+      image: windows-server-2019-nvidia:stable
       shell: bash.exe
 
 commands:
@@ -30,6 +33,18 @@ commands:
 #             git fetch --force origin ${CIRCLE_BRANCH}/merge:merged/${CIRCLE_BRANCH}
 #             git checkout "merged/$CIRCLE_BRANCH"
 #           fi
+  designate_upload_channel:
+    description: "inserts the correct upload channel into ${BASH_ENV}"
+    steps:
+      - run:
+          name: adding UPLOAD_CHANNEL to BASH_ENV
+          command: |
+            our_upload_channel=nightly
+            # On tags upload to test instead
+            if [[ -n "${CIRCLE_TAG}" ]]; then
+              our_upload_channel=test
+            fi
+            echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
 
 binary_common: &binary_common
   parameters:
@@ -59,7 +74,6 @@ binary_common: &binary_common
       default: "pytorch/manylinux-cuda101"
   environment:
     PYTHON_VERSION: << parameters.python_version >>
-    BUILD_VERSION: << parameters.build_version >>
     PYTORCH_VERSION: << parameters.pytorch_version >>
     UNICODE_ABI: << parameters.unicode_abi >>
     CU_VERSION: << parameters.cu_version >>
@@ -118,7 +132,7 @@ jobs:
     - run:
         name: Setup environment
         command: |
-          set -e
+          set -ex
 
           curl -L https://packagecloud.io/circleci/trusty/gpgkey | sudo apt-key add -
           curl -L https://dl.google.com/linux/linux_signing_key.pub | sudo apt-key add -
@@ -152,23 +166,23 @@ jobs:
           sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit=${NVIDIA_CONTAINER_VERSION}
           sudo systemctl restart docker
 
-          DRIVER_FN="NVIDIA-Linux-x86_64-410.104.run"
+          DRIVER_FN="NVIDIA-Linux-x86_64-440.59.run"
           wget "https://s3.amazonaws.com/ossci-linux/nvidia_driver/$DRIVER_FN"
           sudo /bin/bash "$DRIVER_FN" -s --no-drm || (sudo cat /var/log/nvidia-installer.log && false)
           nvidia-smi
 
     - run:
         name: Pull docker image
         command: |
-          set -e
+          set -ex
           export DOCKER_IMAGE=pytorch/conda-cuda
           echo Pulling docker image $DOCKER_IMAGE
           docker pull $DOCKER_IMAGE >/dev/null
 
     - run:
         name: Build and run tests
         command: |
-          set -e
+          set -ex
 
           cd ${HOME}/project/
 
@@ -179,35 +193,79 @@ jobs:
 
   binary_win_conda:
     <<: *binary_common
-    executor:
-      name: win/default
-      shell: bash.exe
+    executor: windows-cpu
     steps:
       - checkout_merge
       - run:
           command: |
-            choco install miniconda3
-            (& "C:\tools\miniconda3\Scripts\conda.exe" "shell.powershell" "hook") | Out-String | Invoke-Expression
+            set -ex
+            source packaging/windows/internal/vc_install_helper.sh
+            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
             conda activate base
             conda install -yq conda-build "conda-package-handling!=1.5.0"
-            bash packaging/build_conda.sh
-          shell: powershell.exe
+            packaging/build_conda.sh
       - store_test_results:
           path: build_results/
 
   binary_win_conda_cuda:
     <<: *binary_common
-    executor: windows-gpu-prototype
+    executor: windows-gpu
+    steps:
+      - checkout_merge
+      - run:
+          command: |
+            set -ex
+            source packaging/windows/internal/vc_install_helper.sh
+            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
+            conda activate base
+            conda install -yq conda-build "conda-package-handling!=1.5.0"
+            packaging/build_conda.sh
+
+  binary_win_conda_release:
+    <<: *binary_common
+    executor: windows-cpu
     steps:
       - checkout_merge
       - run:
+          name: Build conda packages
           command: |
-            choco install miniconda3
-            (& "C:\tools\miniconda3\Scripts\conda.exe" "shell.powershell" "hook") | Out-String | Invoke-Expression
+            set -ex
+            source packaging/windows/internal/vc_install_helper.sh
+            packaging/windows/internal/cuda_install.bat
+            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
             conda activate base
             conda install -yq conda-build "conda-package-handling!=1.5.0"
-            bash packaging/build_conda.sh
-          shell: powershell.exe
+            packaging/build_conda.sh
+            rm /C/tools/miniconda3/conda-bld/win-64/vs${VC_YEAR}*.tar.bz2
+      - store_artifacts:
+          path: C:/tools/miniconda3/conda-bld/win-64
+      - persist_to_workspace:
+          root: C:/tools/miniconda3/conda-bld/win-64
+          paths:
+            - "*"
+      - store_test_results:
+          path: build_results/
+
+  binary_win_wheel_release:
+    <<: *binary_common
+    executor: windows-cpu
+    steps:
+      - checkout_merge
+      - run:
+          name: Build wheel packages
+          command: |
+            set -ex
+            source packaging/windows/internal/vc_install_helper.sh
+            packaging/windows/internal/cuda_install.bat
+            packaging/build_wheel.sh
+      - store_artifacts:
+          path: dist
+      - persist_to_workspace:
+          root: dist
+          paths:
+            - "*"
+      - store_test_results:
+          path: build_results/
 
   binary_macos_wheel:
     <<: *binary_common
@@ -260,12 +318,13 @@ jobs:
     steps:
       - attach_workspace:
           at: ~/workspace
+      - designate_upload_channel
       - run:
           command: |
             # Prevent credential from leaking
             conda install -yq anaconda-client
             set -x
-            anaconda  -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/*.tar.bz2 -u pytorch-nightly --label main --no-progress --force
+            anaconda  -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force
 
   # Requires org-member context
   binary_wheel_upload:
@@ -278,6 +337,7 @@ jobs:
     steps:
       - attach_workspace:
           at: ~/workspace
+      - designate_upload_channel
       - checkout
       - run:
           command: |
@@ -289,7 +349,7 @@ jobs:
             export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
             set -x
             for pkg in ~/workspace/*.whl; do
-              aws s3 cp "$pkg" "s3://pytorch/whl/nightly/<< parameters.subfolder >>" --acl public-read
+              aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/<< parameters.subfolder >>" --acl public-read
             done
 
 
@@ -298,7 +358,11 @@ workflows:
 {%- if True %}
     jobs:
       - circleci_consistency
-      {{ workflows() }}
+      {{ workflows(windows_latest_only=True) }}
+      - binary_linux_conda_cuda:
+          name: torchvision_linux_py3.8_cu102_cuda
+          python_version: "3.8"
+          cu_version: "cu102"
       - binary_win_conda:
           name: torchvision_win_py3.6_cpu
           python_version: "3.6"

.circleci/regenerate.py

@@ -19,16 +19,23 @@
 import os.path
 
 
-def workflows(prefix='', filter_branch=None, upload=False, indentation=6):
+def workflows(prefix='', filter_branch=None, upload=False, indentation=6, windows_latest_only=False):
     w = []
     for btype in ["wheel", "conda"]:
-        for os_type in ["linux", "macos"]:
-            for python_version in ["3.5", "3.6", "3.7", "3.8"]:
-                for cu_version in (["cpu", "cu92", "cu101", "cu102"] if os_type == "linux" else ["cpu"]):
+        for os_type in ["linux", "macos", "win"]:
+            python_versions = ["3.6", "3.7", "3.8"]
+            cu_versions = (["cpu", "cu92", "cu101", "cu102"] if os_type == "linux" or os_type == "win" else ["cpu"])
+            for python_version in python_versions:
+                for cu_version in cu_versions:
                     for unicode in ([False, True] if btype == "wheel" and python_version == "2.7" else [False]):
+                        fb = filter_branch
+                        if windows_latest_only and os_type == "win" and filter_branch is None and \
+                            (python_version != python_versions[-1] or
+                             (cu_version not in [cu_versions[0], cu_versions[-1]])):
+                            fb = "master"
                         w += workflow_pair(
                             btype, os_type, python_version, cu_version,
-                            unicode, prefix, upload, filter_branch=filter_branch)
+                            unicode, prefix, upload, filter_branch=fb)
 
     return indent(indentation, w)
 
@@ -72,15 +79,17 @@ def generate_base_workflow(base_workflow_name, python_version, cu_version,
         "cu_version": cu_version,
     }
 
-    if unicode:
+    if os_type != "win" and unicode:
         d["unicode_abi"] = '1'
 
-    d["wheel_docker_image"] = get_manylinux_image(cu_version)
+    if os_type != "win":
+        d["wheel_docker_image"] = get_manylinux_image(cu_version)
 
     if filter_branch is not None:
         d["filters"] = {"branches": {"only": filter_branch}}
 
-    return {f"binary_{os_type}_{btype}": d}
+    w = f"binary_{os_type}_{btype}_release" if os_type == "win" else f"binary_{os_type}_{btype}"
+    return {w: d}
 
 
 def generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, *, filter_branch=None):
@@ -94,7 +103,16 @@ def generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, *,
         d["subfolder"] = "" if os_type == 'macos' else cu_version + "/"
 
     if filter_branch is not None:
-        d["filters"] = {"branches": {"only": filter_branch}}
+        d["filters"] = {
+            "branches": {
+                "only": filter_branch
+            },
+            "tags": {
+                # Using a raw string here to avoid having to escape
+                # anything
+                "only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
+            }
+        }
 
     return {f"binary_{btype}_upload": d}
 

packaging/build_wheel.sh

@@ -12,4 +12,8 @@ pip_install numpy pyyaml future ninja
 pip_install six
 setup_pip_pytorch_version
 python setup.py clean
-IS_WHEEL=1 python setup.py bdist_wheel
+if [[ "$OSTYPE" == "msys" ]]; then
+    IS_WHEEL=1 "$script_dir/windows/internal/vc_env_helper.bat" python setup.py bdist_wheel
+else
+    IS_WHEEL=1 python setup.py bdist_wheel
+fi

packaging/conda/build_vision.sh

@@ -5,6 +5,10 @@ fi
 
 set -ex
 
+if [[ "$CIRCLECI" == 'true' ]]; then
+    export PATH="/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin:.:$PATH"
+fi
+
 # Function to retry functions that sometimes timeout or have flaky failures
 retry () {
     $*  || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)

packaging/pkg_helpers.bash

@@ -117,6 +117,12 @@ setup_build_version() {
   else
     export BUILD_VERSION="$BUILD_VERSION$VERSION_SUFFIX"
   fi
+
+  # Set build version based on tag if on tag
+  if [[ -n "${CIRCLE_TAG}" ]]; then
+    # Strip tag
+    export BUILD_VERSION="$(echo "${CIRCLE_TAG}" | sed -e 's/^v//' -e 's/-.*$//')"
+  fi
 }
 
 # Set some useful variables for OS X, if applicable
@@ -159,7 +165,7 @@ retry () {
 #
 # Precondition: If Linux, you are in a soumith/manylinux-cuda* Docker image
 setup_wheel_python() {
-  if [[ "$(uname)" == Darwin ]]; then
+  if [[ "$(uname)" == Darwin || "$OSTYPE" == "msys" ]]; then
     eval "$(conda shell.bash hook)"
     conda env remove -n "env$PYTHON_VERSION" || true
     conda create -yn "env$PYTHON_VERSION" python="$PYTHON_VERSION"
@@ -242,6 +248,9 @@ setup_conda_pytorch_constraint() {
     export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==${PYTORCH_VERSION}${PYTORCH_VERSION_SUFFIX}"
     export CONDA_PYTORCH_CONSTRAINT="- pytorch==${PYTORCH_VERSION}${PYTORCH_VERSION_SUFFIX}"
   fi
+  if [[ "$OSTYPE" == msys && "$CU_VERSION" == cu92 ]]; then
+    export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c defaults -c numba/label/dev"
+  fi
 }
 
 # Translate CUDA_VERSION into CUDA_CUDATOOLKIT_CONSTRAINT
@@ -278,8 +287,7 @@ setup_conda_cudatoolkit_constraint() {
 # Build the proper compiler package before building the final package
 setup_visual_studio_constraint() {
   if [[ "$OSTYPE" == "msys" ]]; then
-      export VSTOOLCHAIN_PACKAGE=vs2019
-      export VSDEVCMD_ARGS=''
+      export VSTOOLCHAIN_PACKAGE=vs$VC_YEAR
       conda build $CONDA_CHANNEL_FLAGS --no-anaconda-upload packaging/$VSTOOLCHAIN_PACKAGE
       cp packaging/$VSTOOLCHAIN_PACKAGE/conda_build_config.yaml packaging/torchvision/conda_build_config.yaml
   fi

packaging/vs2019/meta.yaml

@@ -19,27 +19,6 @@ outputs:
       # VS 2019 is binary-compatible with VS 2017/vc 14.1 and 2015/vc14.  Tools are "v142".
       strong:
         - vc{{ vcfeature }}
-    run_exports:
-      - vc {{ vcver }}
     about:
       summary: Activation and version verification of MSVC {{ vcver }} (VS {{ vsyear }}) compiler
       license: BSD 3-clause
-  - name: vs{{ vsyear }}_runtime
-    script: install_runtime.bat
-  - name: vc
-    version: {{ vcver }}
-    track_features:
-      - vc{{ vcfeature }}
-    requirements:
-      run:
-        - {{ pin_subpackage('vs' ~ vsyear ~ '_runtime') }}
-    about:
-      home: https://github.com/conda/conda/wiki/VC-features
-      license: Modified BSD License (3-clause)
-      license_family: BSD
-      summary: A meta-package to track VC features.
-      description: |
-          This metapackage is used to activate vc features without
-          depending on Python.
-      doc_url: https://github.com/conda/conda/wiki/VC-features
-      dev_url: https://github.com/conda/conda/wiki/VC-features

packaging/windows/internal/build_conda.bat

@@ -0,0 +1,15 @@
+if "%VC_YEAR%" == "2017" set VSDEVCMD_ARGS=-vcvars_ver=14.11
+if "%VC_YEAR%" == "2017" powershell packaging/windows/internal/vs2017_install.ps1
+if errorlevel 1 exit /b 1
+
+call packaging/windows/internal/cuda_install.bat
+if errorlevel 1 exit /b 1
+
+call packaging/windows/internal/nightly_defaults.bat Conda
+if errorlevel 1 exit /b 1
+
+set PYTORCH_FINAL_PACKAGE_DIR=%CD%\packaging\windows\output
+if not exist "%PYTORCH_FINAL_PACKAGE_DIR%" mkdir %PYTORCH_FINAL_PACKAGE_DIR%
+
+bash ./packaging/conda/build_vision.sh %CUDA_VERSION% %TORCHVISION_BUILD_VERSION% %TORCHVISION_BUILD_NUMBER%
+if errorlevel 1 exit /b 1