Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ci] Add build script for win #3410

Merged
merged 10 commits into from
Nov 8, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 6 additions & 38 deletions .github/workflows/postsubmit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -194,10 +194,6 @@ jobs:
runs-on: windows-latest
timeout-minutes: 90
steps:
- name: Install 7Zip PowerShell
shell: powershell
run: Install-Module 7Zip4PowerShell -Force -Verbose

- uses: actions/checkout@v2
with:
submodules: "recursive"
Expand All @@ -209,49 +205,21 @@ jobs:
- name: Add msbuild to PATH
uses: microsoft/setup-msbuild@v1.0.2

- name: Download And Install Vulkan
shell: powershell
run: |
Invoke-WebRequest -Uri "https://sdk.lunarg.com/sdk/download/1.2.189.0/windows/VulkanSDK-1.2.189.0-Installer.exe" -OutFile VulkanSDK.exe
$installer = Start-Process -FilePath VulkanSDK.exe -Wait -PassThru -ArgumentList @("/S");
$installer.WaitForExit();

- name: Build
shell: powershell
run: |
$env:Path += ";C:/VulkanSDK/1.2.189.0/Bin"
cd C:\
Remove-item alias:curl
curl --retry 10 --retry-delay 5 https://github.com/taichi-dev/taichi_assets/releases/download/llvm10/taichi-llvm-10.0.0-msvc2019.zip -LO
7z x taichi-llvm-10.0.0-msvc2019.zip -otaichi_llvm
curl --retry 10 --retry-delay 5 https://github.com/taichi-dev/taichi_assets/releases/download/llvm10/clang-10.0.0-win.zip -LO
7z x clang-10.0.0-win.zip -otaichi_clang
$env:PATH = ";C:\taichi_llvm\bin;C:\taichi_clang\bin;" + $env:PATH
clang --version
cd D:\a\taichi\taichi
python -m pip install -r requirements_dev.txt
python -m pip install -r requirements_test.txt
cd python
git fetch origin master
$env:TAICHI_CMAKE_ARGS = $env:CI_SETUP_CMAKE_ARGS
python build.py build
cd ..\dist
$env:WHL = $(dir *.whl)
python -m pip install $env:WHL
env:
PYTHON: C:\hostedtoolcache\windows\Python\3.7.9\x64\python.exe
CI_SETUP_CMAKE_ARGS: -G "Visual Studio 16 2019" -A x64 -DLLVM_DIR=C:\taichi_llvm\lib\cmake\llvm -DTI_WITH_VULKAN:BOOL=ON
VULKAN_SDK: C:/VulkanSDK/1.2.189.0
.\.github\workflows\scripts\win_build.ps1 -installVulkan -install -libsDir C:\

- name: Test
shell: powershell
run: |
$env:PATH = ";C:\taichi_llvm\bin;C:\taichi_clang\bin;" + $env:PATH
. venv\Scripts\activate.ps1
python -c "import taichi"
python examples/algorithm/laplace.py
python bin/taichi diagnose
python bin/taichi changelog
python bin/taichi test -vr2 -t2
ti diagnose
ti changelog
ti test -vr2 -t2
env:
PYTHON: C:\hostedtoolcache\windows\Python\3.7.9\x64\python.exe

Expand Down Expand Up @@ -306,7 +274,7 @@ jobs:
steps:
- uses: actions/checkout@v2
with:
submodules: 'recursive'
submodules: "recursive"

- name: Build & Install
run: |
Expand Down
47 changes: 6 additions & 41 deletions .github/workflows/presubmit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ jobs:
run: |
echo "=============== list modified files ==============="
git diff --name-only @^

echo "========== check paths of modified files =========="
git diff --name-only @^ > files.txt
while IFS= read -r file
Expand Down Expand Up @@ -376,10 +376,6 @@ jobs:
runs-on: windows-latest
timeout-minutes: 90
steps:
- name: Install 7Zip PowerShell
shell: powershell
run: Install-Module 7Zip4PowerShell -Force -Verbose

- uses: actions/checkout@v2
with:
submodules: "recursive"
Expand All @@ -391,45 +387,13 @@ jobs:
- name: Add msbuild to PATH
uses: microsoft/setup-msbuild@v1.0.2

- name: Download And Install Vulkan
shell: powershell
run: |
if ( "${{needs.check_files.outputs.run_job}}" -eq "false" ) {
exit 0
}
Invoke-WebRequest -Uri "https://sdk.lunarg.com/sdk/download/1.2.189.0/windows/VulkanSDK-1.2.189.0-Installer.exe" -OutFile VulkanSDK.exe
$installer = Start-Process -FilePath VulkanSDK.exe -Wait -PassThru -ArgumentList @("/S");
$installer.WaitForExit();

- name: Build
shell: powershell
run: |
if ( "${{needs.check_files.outputs.run_job}}" -eq "false" ) {
exit 0
}
$env:Path += ";C:/VulkanSDK/1.2.189.0/Bin"
cd C:\
Remove-item alias:curl
curl --retry 10 --retry-delay 5 https://github.com/taichi-dev/taichi_assets/releases/download/llvm10/taichi-llvm-10.0.0-msvc2019.zip -LO
7z x taichi-llvm-10.0.0-msvc2019.zip -otaichi_llvm
curl --retry 10 --retry-delay 5 https://github.com/taichi-dev/taichi_assets/releases/download/llvm10/clang-10.0.0-win.zip -LO
7z x clang-10.0.0-win.zip -otaichi_clang
$env:PATH = ";C:\taichi_llvm\bin;C:\taichi_clang\bin;" + $env:PATH
clang --version
cd D:\a\taichi\taichi
python -m pip install -r requirements_dev.txt
python -m pip install -r requirements_test.txt
cd python
git fetch origin master
$env:TAICHI_CMAKE_ARGS = $env:CI_SETUP_CMAKE_ARGS
python build.py build
cd ..\dist
$env:WHL = $(dir *.whl)
python -m pip install $env:WHL
env:
PYTHON: C:\hostedtoolcache\windows\Python\3.7.9\x64\python.exe
CI_SETUP_CMAKE_ARGS: -G "Visual Studio 16 2019" -A x64 -DLLVM_DIR=C:\taichi_llvm\lib\cmake\llvm -DTI_WITH_VULKAN:BOOL=ON
VULKAN_SDK: C:/VulkanSDK/1.2.189.0
.\.github\workflows\scripts\win_build.ps1 -installVulkan -install -libsDir C:\

- name: Test
shell: powershell
Expand All @@ -438,11 +402,12 @@ jobs:
exit 0
}
$env:PATH = ";C:\taichi_llvm\bin;C:\taichi_clang\bin;" + $env:PATH
. venv\Scripts\activate.ps1
python -c "import taichi"
python examples/algorithm/laplace.py
python bin/taichi diagnose
python bin/taichi changelog
python bin/taichi test -vr2 -t2
ti diagnose
ti changelog
ti test -vr2 -t2
env:
PYTHON: C:\hostedtoolcache\windows\Python\3.7.9\x64\python.exe

Expand Down
39 changes: 5 additions & 34 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -288,10 +288,6 @@ jobs:
matrix: ${{ fromJson(needs.matrix_prep.outputs.matrix) }}
runs-on: windows-latest
steps:
- name: Install 7Zip PowerShell
shell: powershell
run: Install-Module 7Zip4PowerShell -Force -Verbose

- uses: actions/checkout@v2
with:
submodules: "recursive"
Expand All @@ -303,39 +299,13 @@ jobs:
- name: Add msbuild to PATH
uses: microsoft/setup-msbuild@v1.0.2

- name: Download And Install Vulkan
shell: powershell
run: |
Invoke-WebRequest -Uri "https://sdk.lunarg.com/sdk/download/1.2.189.0/windows/VulkanSDK-1.2.189.0-Installer.exe" -OutFile VulkanSDK.exe
$installer = Start-Process -FilePath VulkanSDK.exe -Wait -PassThru -ArgumentList @("/S");
$installer.WaitForExit();

- name: Build Python Wheel
shell: powershell
run: |
$env:Path += ";C:/VulkanSDK/1.2.189.0/Bin"
cd C:\
Remove-item alias:curl
curl --retry 10 --retry-delay 5 https://github.com/taichi-dev/taichi_assets/releases/download/llvm10/taichi-llvm-10.0.0-msvc2019.zip -LO
7z x taichi-llvm-10.0.0-msvc2019.zip -otaichi_llvm
curl --retry 10 --retry-delay 5 https://github.com/taichi-dev/taichi_assets/releases/download/llvm10/clang-10.0.0-win.zip -LO
7z x clang-10.0.0-win.zip -otaichi_clang
$env:PATH = ";C:\taichi_llvm\bin;C:\taichi_clang\bin;" + $env:PATH
clang --version
cd D:\a\taichi\taichi
python -m pip install -r requirements_dev.txt
python -m pip install -r requirements_test.txt
cd python
git fetch origin master
$env:TAICHI_CMAKE_ARGS = $env:CI_SETUP_CMAKE_ARGS
python build.py build --project_name $env:PROJECT_NAME
cd ..\dist
$env:WHL = $(dir *.whl)
python -m pip install $env:WHL
.\.github\workflows\scripts\win_build.ps1 -installVulkan -libsDir C:\
venv\Scripts\python -m pip install $(dir dist\*.whl)
env:
CI_SETUP_CMAKE_ARGS: -G "Visual Studio 16 2019" -A x64 -DLLVM_DIR=C:\taichi_llvm\lib\cmake\llvm -DTI_WITH_VULKAN:BOOL=ON
PROJECT_NAME: ${{ matrix.name }}
VULKAN_SDK: C:/VulkanSDK/1.2.189.0

- name: Archive Wheel Artifacts
uses: actions/upload-artifact@v2
Expand All @@ -348,10 +318,11 @@ jobs:
shell: powershell
run: |
$env:PATH = ";C:\taichi_llvm\bin;C:\taichi_clang\bin;" + $env:PATH
. venv\Scripts\activate.ps1
python -c "import taichi"
python examples/algorithm/laplace.py
python bin/taichi diagnose
python bin/taichi test -vr2 -t2
ti diagnose
ti test -vr2 -t2

- name: Upload PyPI
shell: powershell
Expand Down
77 changes: 77 additions & 0 deletions .github/workflows/scripts/win_build.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
# Build script for windows

param (
[switch]$clone = $false,
[switch]$installVulkan = $false,
[switch]$develop = $false,
[switch]$install = $false,
[string]$libsDir = "."
)

$RepoURL = 'https://github.com/taichi-dev/taichi'

function WriteInfo($text) {
Write-Host -ForegroundColor Green "[BUILD] $text"
}

WriteInfo("Install 7Zip")
Install-Module 7Zip4PowerShell -Force -Verbose -Scope CurrentUser

if ($clone) {
WriteInfo("Clone the repository")
git clone --recurse-submodules $RepoURL
Set-Location .\taichi
}

$libsDir = (Resolve-Path $libsDir).Path

if (-not (Test-Path $libsDir)) {
New-Item -ItemType Directory -Path $libsDir
}
Push-Location $libsDir
WriteInfo("Download and extract LLVM")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note that the LLVM/clang/Vulkan libs usually are usually downloaded once and placed somewhere outside the repo dir/build dir, (i.e., a more stable place, e.g. C:\ThirdParty\Libs). I'd suggest that we add one additional flag to hold all these required dependent libs, WDYT?

In addition, let's make the default behavior not to download these libs?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the lib location can be specified with -build option, and if the library exists, won't download again.

So -build C:\ThirdParty\Libs can do the trick.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see, then let's give it a more obvious name, e.g. libsdir, depsdir, ...

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The name is changed to libdir

Copy link
Member

@k-ye k-ye Nov 8, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Huh, I guess the naming convention here is $libsDir? Also, the default value should be empty (or at least not _build)

Copy link
Collaborator Author

@frostming frostming Nov 8, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've updated the name and default to the current location (.) since empty isn't a valid path parameter

if (-not (Test-Path "taichi_llvm")) {
curl.exe --retry 10 --retry-delay 5 https://github.com/taichi-dev/taichi_assets/releases/download/llvm10/taichi-llvm-10.0.0-msvc2019.zip -LO
7z x taichi-llvm-10.0.0-msvc2019.zip -otaichi_llvm
}
WriteInfo("Download and extract Clang")
if (-not (Test-Path "taichi_clang")) {
curl.exe --retry 10 --retry-delay 5 https://github.com/taichi-dev/taichi_assets/releases/download/llvm10/clang-10.0.0-win.zip -LO
7z x clang-10.0.0-win.zip -otaichi_clang
}
$env:PATH = "$libsDir\taichi_llvm\bin;$libsDir\taichi_clang\bin;$env:PATH"
$env:TAICHI_CMAKE_ARGS = "-G 'Visual Studio 16 2019' -A x64 -DLLVM_DIR=$libsDir\taichi_llvm\lib\cmake\llvm"
if ($installVulkan) {
WriteInfo("Download and install Vulkan")
if (-not (Test-Path "VulkanSDK.exe")) {
curl.exe --retry 10 --retry-delay 5 https://sdk.lunarg.com/sdk/download/1.2.189.0/windows/VulkanSDK-1.2.189.0-Installer.exe -Lo VulkanSDK.exe
}
$installer = Start-Process -FilePath VulkanSDK.exe -Wait -PassThru -ArgumentList @("/S");
$installer.WaitForExit();
$env:VULKAN_SDK = "$libsDir\VulkanSDK\1.2.189.0"
$env:PATH += ";$env:VULKAN_SDK\Bin"
$env:TAICHI_CMAKE_ARGS += " -DTI_WITH_VULKAN:BOOL=ON"
}

Pop-Location
clang --version

WriteInfo("Setting up Python environment")
python -m venv venv
. venv\Scripts\activate.ps1
python -m pip install wheel
python -m pip install -r requirements_dev.txt
python -m pip install -r requirements_test.txt
WriteInfo("Building Taichi")
$env:CXX = "$libsDir\taichi_clang\bin\clang++.exe"
if ($install) {
if ($develop) {
python -m pip install -v -e .
} else {
python -m pip install -v .
}
WriteInfo("Build and install finished")
} else {
python setup.py bdist_wheel
WriteInfo("Build finished")
}
10 changes: 7 additions & 3 deletions cmake/PythonNumpyPybind11.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,14 @@ execute_process(COMMAND ${PYTHON_EXECUTABLE} -c
sys.stdout.write(str(sys.version_info[1]))"
OUTPUT_VARIABLE PYTHON_MINOR_VERSION)


if (WIN32)
link_directories(${PYTHON_LIBRARY_DIR}/../../libs)
set(PYTHON_LIBRARIES ${PYTHON_LIBRARY_DIR}/../../libs/python3.lib)
set(PYTHON_LIBRARIES ${PYTHON_LIBRARY_DIR}/../../libs/python3${PYTHON_MINOR_VERSION}.lib)
execute_process(COMMAND ${PYTHON_EXECUTABLE} -c
"import sys;sys.stdout.write(sys.base_prefix.replace('\\\\', '/'))"
OUTPUT_VARIABLE PYTHON_BASE_PREFIX)
link_directories(${PYTHON_BASE_PREFIX}/libs)
set(PYTHON_LIBRARIES ${PYTHON_BASE_PREFIX}/libs/python3.lib)
set(PYTHON_LIBRARIES ${PYTHON_BASE_PREFIX}/libs/python3${PYTHON_MINOR_VERSION}.lib)
else()
find_library(PYTHON_LIBRARY NAMES python${PYTHON_VERSION} python${PYTHON_VERSION}m PATHS ${PYTHON_LIBRARY_DIR}
NO_DEFAULT_PATH NO_SYSTEM_ENVIRONMENT_PATH PATH_SUFFIXES x86_64-linux-gnu)
Expand Down