Skip to content

Commit

Permalink
ENH: Azure pipelines script refactoring
Browse files Browse the repository at this point in the history
Split Linux Python packages to avoid timeout issues.

Remove ITKPythonGitTag from .yml variable. It should be defined in the pipeline
variables section when configuring a new pipeline.

Use ITKv5.0.1 in setup.py until ITKPythonBuilds for v5.1.0 are ready.

Setup self hosted pipeline to build with CUDA.
Building on manylinux image requires CUDA_NVCC_FLAGS="-std=c++11".

WARNING: CUDA packages for Linux build successfully but fail at runtime
because of a known issue with auditwheel. See the following for
reference: pypa/auditwheel#159
The docker image used to build packages should be updated with a
custom version of patchelf to work around the issue.
LucasGandel committed Jan 29, 2020
1 parent f02de5c commit 090d8f1
Showing 11 changed files with 211 additions and 80 deletions.
78 changes: 0 additions & 78 deletions azure-pipelines-python.yml

This file was deleted.

2 changes: 2 additions & 0 deletions cmake/FindCUDA_wrap.cmake
Original file line number Diff line number Diff line change
@@ -64,6 +64,8 @@ else()
)
endif()

list(APPEND CUDA_NVCC_FLAGS "-std=c++11")

if(CUDA_FOUND)
try_run(RUN_RESULT_VAR COMPILE_RESULT_VAR
${CMAKE_BINARY_DIR}
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
@@ -44,6 +44,6 @@
keywords='RTK Reconstruction Toolkit',
url=r'https://www.openrtk.org/',
install_requires=[
r'itk>=5.1'
r'itk>=5.0.1'
]
)
Original file line number Diff line number Diff line change
@@ -127,4 +127,3 @@ jobs:
- script: |
$(compilerInitialization)
ctest -j 2 -V -S dashboard.cmake
70 changes: 70 additions & 0 deletions test/ContinuousIntegration/AzurePipelinesPythonPackages.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
variables:
CMakeBuildType: Release

trigger:
batch: true
branches:
include:
- master

jobs:

# Linux Python 3.5
- job: 'PackageLinux_cp35'
timeoutInMinutes: 0
cancelTimeoutInMinutes: 300
variables:
PythonVersion: cp35
displayName: "Build Linux Python 3.5 packages"
pool:
vmImage: 'Ubuntu-16.04'
steps:
- template: Common/AzurePipelinesLinuxPython_Common.yml

# Linux Python 3.6
- job: 'PackageLinux_cp36'
timeoutInMinutes: 0
cancelTimeoutInMinutes: 300
variables:
PythonVersion: cp36
displayName: "Build Linux Python 3.6 packages"
pool:
vmImage: 'Ubuntu-16.04'
steps:
- template: Common/AzurePipelinesLinuxPython_Common.yml

# Linux Python 3.7
- job: 'PackageLinux_cp37'
timeoutInMinutes: 0
cancelTimeoutInMinutes: 300
variables:
PythonVersion: cp37
displayName: "Build Linux Python 3.7 packages"
pool:
vmImage: 'Ubuntu-16.04'
steps:
- template: Common/AzurePipelinesLinuxPython_Common.yml


# MacOS Python
- job: 'PackageMacOS'
displayName: "Build macOS Python packages"
timeoutInMinutes: 0
cancelTimeoutInMinutes: 300
pool:
vmImage: 'macos-10.14'

steps:
- template: Common/AzurePipelinesMacOSPython_Common.yml


# Windows Python
- job: 'PackageWindows'
displayName: "Build Windows Python packages"
timeoutInMinutes: 0
cancelTimeoutInMinutes: 300
pool:
vmImage: 'vs2017-win2016'

steps:
- template: Common/AzurePipelinesWindowsPython_Common.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
variables:
CMakeBuildType: Release

trigger:
batch: true
branches:
include:
- master

jobs:

# Linux Python 3.6 CUDA
- job: 'PackageLinux'
timeoutInMinutes: 0
cancelTimeoutInMinutes: 300
variables:
PythonVersion: cp36
displayName: "Build Linux Python packages"
pool:
name: 'Default'
demands: Agent.OS -equals Linux
steps:
- template: Common/AzurePipelinesLinuxPython_Common_CUDA.yml
- task: DeleteFiles@1
inputs:
SourceFolder: $(Build.SourcesDirectory)
Contents: '**/*'

# MacOS Python CUDA
- job: 'PackageMacOS'
displayName: "Build macOS Python packages"
timeoutInMinutes: 0
cancelTimeoutInMinutes: 300
pool:
name: 'Default'
demands: Agent.OS -equals Darwin

steps:
- template: Common/AzurePipelinesMacOSPython_Common.yml
- task: DeleteFiles@1
inputs:
SourceFolder: $(Build.SourcesDirectory)
Contents: '**/*'

# Windows Python CUDA
- job: 'PackageWindows'
displayName: "Build Windows Python packages"
timeoutInMinutes: 0
cancelTimeoutInMinutes: 300
pool:
name: 'Default'
demands: Agent.OS -equals Windows_NT

steps:
- template: Common/AzurePipelinesWindowsPython_Common.yml
- task: DeleteFiles@1
inputs:
SourceFolder: $(Build.SourcesDirectory)
Contents: '**/*'
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# Build Linux Python packages
steps:
- template: AzurePipelinesLinuxPython_DownloadCache.yml

- script: ./ITKPythonPackage/scripts/dockcross-manylinux-build-module-wheels.sh $(PythonVersion)
displayName: 'Build Python Package'

- task: PublishPipelineArtifact@0
inputs:
artifactName: 'LinuxWheels_$(PythonVersion)'
targetPath: './dist'
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# Build Linux Python packages
steps:
- template: AzurePipelinesLinuxPython_DownloadCache.yml

- script: |
export ITK_PACKAGE_VERSION=$(ITKPythonGitTag)
docker pull lucasgandel/manylinux1-x64-cuda:10.1
docker run lucasgandel/manylinux1-x64-cuda:10.1 > /tmp/dockcross-manylinux-x64
chmod u+x /tmp/dockcross-manylinux-x64
script_dir=$(pwd)/ITKPythonPackage/scripts/
mkdir -p dist
DOCKER_ARGS="-v $(pwd)/dist:/work/dist/ -v $script_dir/..:/ITKPythonPackage -v $(pwd)/tools:/tools"
/tmp/dockcross-manylinux-x64 -a "$DOCKER_ARGS" \
"/ITKPythonPackage/scripts/internal/manylinux-build-module-wheels.sh" "$(PythonVersion)"
displayName: 'Build Python packages'

- task: PublishPipelineArtifact@0
inputs:
artifactName: 'LinuxWheels-CUDA-$(PythonVersion)'
targetPath: './dist'
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# Fetch Linux Python Packages cache
steps:
- script: |
export ITK_PACKAGE_VERSION=$(ITKPythonGitTag)
curl https://data.kitware.com/api/v1/file/592dd8068d777f16d01e1a92/download -o zstd-1.2.0-linux.tar.gz
gunzip -d zstd-1.2.0-linux.tar.gz
tar xf zstd-1.2.0-linux.tar
curl -L https://github.com/InsightSoftwareConsortium/ITKPythonBuilds/releases/download/${ITK_PACKAGE_VERSION:=v5.0.1}/ITKPythonBuilds-linux.tar.zst -O
./zstd-1.2.0-linux/bin/unzstd ITKPythonBuilds-linux.tar.zst -o ITKPythonBuilds-linux.tar
tar xf ITKPythonBuilds-linux.tar
mkdir tools
curl https://data.kitware.com/api/v1/file/5c0aa4b18d777f2179dd0a71/download -o doxygen-1.8.11.linux.bin.tar.gz
tar -xvzf doxygen-1.8.11.linux.bin.tar.gz -C tools
displayName: 'Fetch build cache'
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# Build MacOS Python packages
steps:
- script: |
curl -L https://raw.githubusercontent.com/InsightSoftwareConsortium/ITKPythonPackage/master/scripts/macpython-download-cache-and-build-module-wheels.sh -O
chmod u+x macpython-download-cache-and-build-module-wheels.sh
displayName: 'Fetch build script'
- script: |
export ITK_PACKAGE_VERSION=$(ITKPythonGitTag)
./macpython-download-cache-and-build-module-wheels.sh
displayName: 'Build Python packages'
- task: PublishPipelineArtifact@0
inputs:
artifactName: 'MacOSWheels'
targetPath: './dist'
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# Build Windows Python packages
steps:
- script: |
curl -L https://raw.githubusercontent.com/InsightSoftwareConsortium/ITKPythonPackage/master/scripts/windows-download-cache-and-build-module-wheels.ps1 -O
displayName: 'Fetch build script'
- script: |
call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
set ITK_PACKAGE_VERSION=$(ITKPythonGitTag)
set CC=cl.exe
set CXX=cl.exe
powershell.exe -file .\windows-download-cache-and-build-module-wheels.ps1
displayName: 'Build Python packages'
- task: PublishPipelineArtifact@0
inputs:
artifactName: 'WindowsWheels'
targetPath: './dist'

0 comments on commit 090d8f1

Please sign in to comment.