Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Try to Use Ninja to Speed Up Build #666

Closed
wants to merge 11 commits into from
Closed

Try to Use Ninja to Speed Up Build #666

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
7365e5d
ci(action-install-linux.sh): install ninja build
WhiredPlanck Jun 25, 2023
7ba5cca
ci(macos-build.yml): install ninja-build tool
WhiredPlanck Jun 25, 2023
f87b418
ci(linux-build.yml): export CMAKE_GENERATOR as Ninja
WhiredPlanck Jun 25, 2023
a1f8ac8
ci(macos-build.yml): try to use ninja build on macos
WhiredPlanck Jun 25, 2023
970f3ae
build(Makefile,deps.mk): add flag to control parallel build
WhiredPlanck Jun 25, 2023
d770a1f
feat(env.bat.template...): allow to set desired makefile generator on…
WhiredPlanck Jun 26, 2023
709bbae
ci(windows-build.yml): try to use ninja to build
WhiredPlanck Jun 26, 2023
19407b5
build(build.bat): use lowercase target name
WhiredPlanck Jun 26, 2023
c3ba2c3
build(build.bat): explicitly specify CMAKE_BUILD_TYPE
WhiredPlanck Jun 26, 2023
242ecbc
build(build.bat): add alternative run path of rime_test.exe
WhiredPlanck Jun 26, 2023
99d6b69
ci(workflows): minor refactor to workflow steps
WhiredPlanck Jun 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .github/workflows/linux-build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,4 +25,6 @@ jobs:
run: ./action-install-linux.sh

- name: Build and test
run: make test
run: |
export CMAKE_GENERATOR=Ninja
make test
10 changes: 7 additions & 3 deletions .github/workflows/macos-build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,13 @@ jobs:
repository: ${{ inputs.repository }}
submodules: recursive

- name: Install ninja-build tool
uses: seanmiddleditch/gha-setup-ninja@v3

- name: Configure build environment
run: |
echo git_ref_name="$(git describe --always)" >> $GITHUB_ENV
echo CMAKE_GENERATOR="Ninja" >> $GITHUB_ENV

- name: Configure build variant
if: contains(inputs.build_variant, '-universal')
Expand Down Expand Up @@ -71,17 +75,17 @@ jobs:

- name: Build dependencies
if: steps.cache-deps.outputs.cache-hit != 'true'
run: make xcode/deps
run: make deps

- name: Install Rime plugins
run: ./action-install-plugins-macos.sh

- name: Build and test
run: make xcode/test
run: make test

- name: Create distributable
run: |
make xcode/dist
make install
tar -cjvf rime-${{ env.git_ref_name }}-${{ runner.os }}.tar.bz2 \
dist version-info.txt
tar -cjvf rime-deps-${{ env.git_ref_name }}-${{ runner.os }}.tar.bz2 \
Expand Down
8 changes: 8 additions & 0 deletions .github/workflows/windows-build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,15 @@ jobs:
repository: ${{ inputs.repository }}
submodules: recursive

- name: Install ninja-build tool
uses: seanmiddleditch/gha-setup-ninja@v3

- name: Configure build environment
run: |
copy env.vs2022.bat env.bat
$git_ref_name = git describe --always
echo "git_ref_name=$git_ref_name" >> $env:GITHUB_ENV
echo "CMAKE_GENERATOR=Ninja" >> $env:GITHUB_ENV

- name: Cache Boost
id: cache-boost
Expand Down Expand Up @@ -65,12 +69,16 @@ jobs:

- name: Build dependencies
if: steps.cache-deps.outputs.cache-hit != 'true'
env:
CMAKE_GENERATOR: Ninja
run: .\build.bat deps

- name: Install Rime plugins
run: .\action-install-plugins-windows.bat

- name: Build and test
env:
CMAKE_GENERATOR: Ninja
run: .\build.bat test

- name: Create distributable
Expand Down
30 changes: 30 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,30 @@
RIME_ROOT ?= $(CURDIR)

ifeq ($(shell uname),Darwin) # for macOS
prefix ?= $(RIME_ROOT)/dist

ifdef BOOST_ROOT
CMAKE_BOOST_OPTIONS = -DBoost_NO_BOOST_CMAKE=TRUE \
-DBOOST_ROOT="$(BOOST_ROOT)"
endif

# https://cmake.org/cmake/help/latest/variable/CMAKE_OSX_SYSROOT.html
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These lines repeats scripts in xcode.mk. Can Makefile replace xcode.mk after this change?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@lotem I think the answer is yes. We can merge Makefile and xcode.mk since the latter just specify CMAKE_GENERATOR as Xcode and nothing else is different with the former.

export SDKROOT ?= $(shell xcrun --sdk macosx --show-sdk-path)

# https://cmake.org/cmake/help/latest/envvar/MACOSX_DEPLOYMENT_TARGET.html
export MACOSX_DEPLOYMENT_TARGET ?= 10.13

ifdef BUILD_UNIVERSAL
# https://cmake.org/cmake/help/latest/envvar/CMAKE_OSX_ARCHITECTURES.html
export CMAKE_OSX_ARCHITECTURES = arm64;x86_64
endif

# boost::locale library from homebrew links to homebrewed icu4c libraries
icu_prefix = $(shell brew --prefix)/opt/icu4c

else # for Linux
prefix ?= $(DESTDIR)/usr
endif

debug install-debug uninstall-debug test-debug: build ?= debug
build ?= build
Expand Down Expand Up @@ -34,6 +58,12 @@ librime: release
install-librime: install
uninstall-librime: uninstall

ifdef NOPARALLEL
export MAKEFLAGS+=" -j1 "
else
export MAKEFLAGS+=" -j$(( $(nproc) + 1)) "
endif

librime-static:
cmake . -B$(build) \
-DCMAKE_INSTALL_PREFIX=$(prefix) \
Expand Down
1 change: 1 addition & 0 deletions action-install-linux.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ sudo apt update -y
# https://bugs.launchpad.net/ubuntu/+source/google-glog/+bug/1991919
# https://github.com/kadalu-tech/pkgs/pull/2/files#r1001042597
sudo apt install -y libunwind-dev
sudo apt install -y ninja-build
sudo apt install -y ${dep_packages[@]}
make deps/gtest
make -C deps/opencc build
Expand Down
18 changes: 10 additions & 8 deletions build.bat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,7 @@ if defined PLATFORM_TOOLSET (
)
set deps_cmake_flags=%common_cmake_flags%^
-DCMAKE_CONFIGURATION_TYPES:STRING="%build_config%"^
-DCMAKE_BUILD_TYPE:STRING="%build_config%"^
-DCMAKE_CXX_FLAGS_RELEASE:STRING="/MT /O2 /Ob2 /DNDEBUG"^
-DCMAKE_C_FLAGS_RELEASE:STRING="/MT /O2 /Ob2 /DNDEBUG"^
-DCMAKE_CXX_FLAGS_DEBUG:STRING="/MTd /Od"^
Expand All @@ -209,7 +210,7 @@ if %build_deps% == 1 (
-DWITH_GFLAGS:BOOL=OFF^
-DCMAKE_MSVC_RUNTIME_LIBRARY="MultiThreaded$<$<CONFIG:Debug>:Debug>"
if errorlevel 1 goto error
cmake --build cmake-%build_dir% --config %build_config% --target INSTALL
cmake --build cmake-%build_dir% --config %build_config% --target install
if errorlevel 1 goto error
popd

Expand All @@ -219,7 +220,7 @@ if %build_deps% == 1 (
-DLEVELDB_BUILD_BENCHMARKS:BOOL=OFF^
-DLEVELDB_BUILD_TESTS:BOOL=OFF
if errorlevel 1 goto error
cmake --build %build_dir% --config %build_config% --target INSTALL
cmake --build %build_dir% --config %build_config% --target install
if errorlevel 1 goto error
popd

Expand All @@ -232,7 +233,7 @@ if %build_deps% == 1 (
-DYAML_CPP_BUILD_TESTS:BOOL=OFF^
-DYAML_CPP_BUILD_TOOLS:BOOL=OFF
if errorlevel 1 goto error
cmake --build %build_dir% --config %build_config% --target INSTALL
cmake --build %build_dir% --config %build_config% --target install
if errorlevel 1 goto error
popd

Expand All @@ -241,15 +242,15 @@ if %build_deps% == 1 (
cmake . -B%build_dir% %deps_cmake_flags%^
-DBUILD_GMOCK:BOOL=OFF
if errorlevel 1 goto error
cmake --build %build_dir% --config %build_config% --target INSTALL
cmake --build %build_dir% --config %build_config% --target install
if errorlevel 1 goto error
popd

echo building marisa.
pushd deps\marisa-trie
cmake .. -B%build_dir% %deps_cmake_flags%
if errorlevel 1 goto error
cmake --build %build_dir% --config %build_config% --target INSTALL
cmake --build %build_dir% --config %build_config% --target install
if errorlevel 1 goto error
popd

Expand All @@ -259,7 +260,7 @@ if %build_deps% == 1 (
-DBUILD_SHARED_LIBS=OFF^
-DBUILD_TESTING=OFF
if errorlevel 1 goto error
cmake --build %build_dir% --config %build_config% --target INSTALL
cmake --build %build_dir% --config %build_config% --target install
if errorlevel 1 goto error
popd
)
Expand All @@ -273,6 +274,7 @@ set rime_cmake_flags=%common_cmake_flags%^
-DENABLE_LOGGING=%enable_logging%^
-DBOOST_USE_CXX11=ON^
-DCMAKE_CONFIGURATION_TYPES="%build_config%"^
-DCMAKE_BUILD_TYPE="%build_config%"^
-DCMAKE_INSTALL_PREFIX:PATH="%RIME_ROOT%\dist"

echo on
Expand All @@ -284,14 +286,14 @@ echo.
echo building librime.
echo.
echo on
cmake --build %build_dir% --config %build_config% --target INSTALL
cmake --build %build_dir% --config %build_config% --target install
@echo off
if errorlevel 1 goto error

if "%build_test%" == "ON" (
copy /y dist\lib\rime.dll build\test
pushd build\test
.\Release\rime_test.exe || goto error
.\Release\rime_test.exe || .\rime_test.exe || goto error
popd
)

Expand Down
6 changes: 6 additions & 0 deletions deps.mk
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,12 @@
rime_root = $(CURDIR)
src_dir = $(rime_root)/deps

ifdef NOPARALLEL
export MAKEFLAGS+=" -j1 "
else
export MAKEFLAGS+=" -j$(( $(nproc) + 1)) "
endif

glog: build ?= cmake-build
build ?= build

Expand Down
16 changes: 12 additions & 4 deletions env.bat.template
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,18 @@ rem REQUIRED: path to Boost source directory
if not defined BOOST_ROOT set BOOST_ROOT=%RIME_ROOT%\deps\boost_1_78_0

rem architecture, Visual Studio version and platform toolset
set ARCH=Win32
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Some of these envars are not set in the else-branch. That's probably not okay because build.bat uses these variables.

Copy link
Contributor Author

@WhiredPlanck WhiredPlanck Jul 4, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@lotem These envars like ARCH here only make sense when CMAKE_GENERATOR is something like Visual Studio xx xxxx. I think there are enough refactors in `build.bat to adopt these changes.

set BJAM_TOOLSET=msvc-14.2
set CMAKE_GENERATOR="Visual Studio 16 2019"
set PLATFORM_TOOLSET=v142
if not defined CMAKE_GENERATOR (
set ARCH=Win32
set BJAM_TOOLSET=msvc-14.2
set CMAKE_GENERATOR="Visual Studio 16 2019"
set PLATFORM_TOOLSET=v142
) else (
set VSVEROPT="-version 16,17"
call "%~dp0env.vswhere.bat" x86
set VSVEROPT=
set CXX=cl
set CC=cl
)

rem OPTIONAL: path to additional build tools
rem set DEVTOOLS_PATH=%ProgramFiles%\Git\cmd;%ProgramFiles%\CMake\bin;C:\Python27;
16 changes: 12 additions & 4 deletions env.vs2017_xp.bat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,18 @@ rem REQUIRED: path to Boost source directory
if not defined BOOST_ROOT set BOOST_ROOT=%RIME_ROOT%\deps\boost_1_69_0

rem architecture, Visual Studio version and platform toolset
set ARCH=Win32
set BJAM_TOOLSET=msvc-14.1
set CMAKE_GENERATOR="Visual Studio 15 2017"
set PLATFORM_TOOLSET=v141_xp
if not defined CMAKE_GENERATOR (
set ARCH=Win32
set BJAM_TOOLSET=msvc-14.1
set CMAKE_GENERATOR="Visual Studio 15 2017"
set PLATFORM_TOOLSET=v141_xp
) else (
set "VSVEROPT=-version [15.0^,16.0^)"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What does the else-block do? Why is it on the defined CMAKE_GENERATOR branch?

Copy link
Contributor Author

@WhiredPlanck WhiredPlanck Jul 4, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If CMAKE_GENERATOR is defined outside the scripts, it may not be Visual Studio xx xxxx. To preserve the original settings, I split out a else-block for non-Visual-Studio generator to run a script which call vswhere.exe to find the Visual Studio's install location and call vcvarsall.bat to set almost all environment variables we need.
In a word, if one don't specify CMAKE_GENERATOR before calling env.bat and build.bat to build, it will use our original settings to build, otherwise the scripts will find and set everything we need and build with the specified generator.

call ./env.vswhere.bat x86
set VSVEROPT=
set CXX=cl
set CC=cl
)

rem OPTIONAL: path to additional build tools
rem set DEVTOOLS_PATH=%ProgramFiles%\Git\cmd;%ProgramFiles%\CMake\bin;C:\Python27;
16 changes: 12 additions & 4 deletions env.vs2019.bat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,18 @@ rem REQUIRED: path to Boost source directory
if not defined BOOST_ROOT set BOOST_ROOT=%RIME_ROOT%\deps\boost_1_78_0

rem architecture, Visual Studio version and platform toolset
set ARCH=Win32
set BJAM_TOOLSET=msvc-14.2
set CMAKE_GENERATOR="Visual Studio 16 2019"
set PLATFORM_TOOLSET=v142
if not defined CMAKE_GENERATOR (
set ARCH=Win32
set BJAM_TOOLSET=msvc-14.2
set CMAKE_GENERATOR="Visual Studio 16 2019"
set PLATFORM_TOOLSET=v142
) else (
set "VSVEROPT=-version [16.0^,17.0^)"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How about creating a separate env.bat file with the else branch.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@lotem No need actually. Else-block plays the same role with if-block.

call ./env.vswhere.bat x86
set VSVEROPT=
set CXX=cl
set CC=cl
)

rem OPTIONAL: path to additional build tools
rem set DEVTOOLS_PATH=%ProgramFiles%\Git\cmd;%ProgramFiles%\CMake\bin;C:\Python27;
16 changes: 12 additions & 4 deletions env.vs2022.bat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,18 @@ rem REQUIRED: path to Boost source directory
if not defined BOOST_ROOT set BOOST_ROOT=%RIME_ROOT%\deps\boost_1_78_0

rem architecture, Visual Studio version and platform toolset
set ARCH=Win32
set BJAM_TOOLSET=msvc-14.3
set CMAKE_GENERATOR="Visual Studio 17 2022"
set PLATFORM_TOOLSET=v143
if not defined CMAKE_GENERATOR (
set ARCH=Win32
set BJAM_TOOLSET=msvc-14.3
set CMAKE_GENERATOR="Visual Studio 17 2022"
set PLATFORM_TOOLSET=v143
) else (
set "VSVEROPT=-version [17.0^,18.0^)"
call ./env.vswhere.bat x86
set VSVEROPT=
set CXX=cl
set CC=cl
)

rem OPTIONAL: path to additional build tools
rem set DEVTOOLS_PATH=%ProgramFiles%\Git\cmd;%ProgramFiles%\CMake\bin;C:\Python27;
73 changes: 73 additions & 0 deletions env.vswhere.bat
View file
Open in desktop
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What does this script do?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@lotem To find where Visual Studio installed in and set some envars we need.

Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
@echo off
rem This file is taken from vim project.
rem To be used on MS-Windows for Visual C++ 2017 or later.
rem See INSTALLpc.txt for information.
rem
rem Usage:
rem For x86 builds run this with "x86" option:
rem msvc-latest x86
rem For x64 builds run this with "x86_amd64" option or "x64" option:
rem msvc-latest x86_amd64
rem msvc-latest x64
rem
rem Optional environment variables:
rem VSWHERE:
rem Full path to vswhere.exe.
rem VSVEROPT:
rem Option to search specific version of Visual Studio.
rem Default: -latest
rem To search VS2017:
rem set "VSVEROPT=-version [15.0^,16.0^)"
rem To search VS2019:
rem set "VSVEROPT=-version [16.0^,17.0^)"
rem To search VS2022:
rem set "VSVEROPT=-version [17.0^,18.0^)"

if "%VSWHERE%"=="" (
set "VSWHERE=%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe"
set VSWHERE_SET=yes
)
if not exist "%VSWHERE%" (
echo Error: vswhere not found.
set VSWHERE=
set VSWHERE_SET=
exit /b 1
)

if "%VSVEROPT%"=="" (
set VSVEROPT=-latest
set VSVEROPT_SET=yes
)

rem Search Visual Studio Community, Professional or above.
for /f "usebackq tokens=*" %%i in (`"%VSWHERE%" %VSVEROPT% -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do (
set InstallDir=%%i
)
if exist "%InstallDir%\VC\Auxiliary\Build\vcvarsall.bat" (
call "%InstallDir%\VC\Auxiliary\Build\vcvarsall.bat" %*
goto done
)

rem Search Visual Studio 2017 Express.
rem (Visual Studio 2017 Express uses different component IDs.)
for /f "usebackq tokens=*" %%i in (`"%VSWHERE%" %VSVEROPT% -products Microsoft.VisualStudio.Product.WDExpress -property installationPath`) do (
set InstallDir=%%i
)
if exist "%InstallDir%\VC\Auxiliary\Build\vcvarsall.bat" (
call "%InstallDir%\VC\Auxiliary\Build\vcvarsall.bat" %*
) else (
echo Error: vcvarsall.bat not found.
rem Set ERRORLEVEL to 1.
call
)

:done
if "%VSWHERE_SET%"=="yes" (
set VSWHERE=
set VSWHERE_SET=
)
if "%VSVEROPT_SET%"=="yes" (
set VSVEROPT=
set VSVEROPT_SET=
)
set InstallDir=