Skip to content

Commit

Permalink
Update coredistools from LLVM 13.0.1 to 16.0.1
Browse files Browse the repository at this point in the history
1. LLVM doesn't release a full set of binary drops, so we need to
build llvm-tblgen on some platforms, namely Mac.
2. Build using standard CBL-Mariner Docker containers (used by
dotnet/runtime as well). This also converts linux-x64 builds to be
container-based cross builds.
  • Loading branch information
BruceForstall committed Apr 18, 2023
1 parent c38952c commit d07dbed
Show file tree
Hide file tree
Showing 5 changed files with 184 additions and 65 deletions.
40 changes: 29 additions & 11 deletions build-coredistools.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,29 @@ EnsureCrossRootfsDirectoryExists () {
CMakeOSXArchitectures=
LLVMTargetsToBuild="AArch64;ARM;X86"

# Figure out which `strip` to use. Prefer `llvm-strip` if it is available.
# `llvm-strip` is available in CBL-Mariner container; `strip` is available on macOS.
StripTool=$(command -v llvm-strip)
if [ -z "$StripTool" ]; then
StripTool=$(command -v strip)
if [ -z "$StripTool" ]; then
echo "Strip tool not found"
exit 1
fi
fi

TblGenTool=$(command -v llvm-tblgen)
if [ -z "$TblGenTool" ]; then
echo "llvm-tblgen tool not found"
exit 1
fi

case "$TargetOSArchitecture" in
linux-arm)
CMakeCrossCompiling=ON
LLVMDefaultTargetTriple=thumbv7-linux-gnueabihf
LLVMHostTriple=arm-linux-gnueabihf
LLVMTargetsToBuild=ARM
LLVMTargetsToBuild="ARM"
EnsureCrossRootfsDirectoryExists
;;

Expand All @@ -29,8 +46,9 @@ case "$TargetOSArchitecture" in
;;

linux-x64)
CMakeCrossCompiling=OFF
CMakeCrossCompiling=ON
LLVMHostTriple=x86_64-linux-gnu
EnsureCrossRootfsDirectoryExists
;;

linux-loongarch64)
Expand Down Expand Up @@ -63,7 +81,7 @@ SourcesDirectory=$RootDirectory/src
BinariesDirectory=$RootDirectory/obj/$TargetOSArchitecture
StagingDirectory=$RootDirectory/artifacts/$TargetOSArchitecture

which cmake >/dev/null 2>&1
command -v cmake >/dev/null 2>&1

if [ "$?" -ne 0 ]; then
echo "ERROR: cmake is not found in the PATH"
Expand All @@ -81,20 +99,20 @@ if [ -z "$CrossRootfsDirectory" ]; then
-G "Unix Makefiles" \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_CROSSCOMPILING=$CMakeCrossCompiling \
-DCMAKE_C_COMPILER=$(which clang) \
-DCMAKE_C_COMPILER=$(command -v clang) \
-DCMAKE_C_FLAGS="-target $LLVMHostTriple" \
-DCMAKE_CXX_COMPILER=$(which clang++) \
-DCMAKE_CXX_COMPILER=$(command -v clang++) \
-DCMAKE_CXX_FLAGS="-target $LLVMHostTriple" \
-DCMAKE_INSTALL_PREFIX=$StagingDirectory \
-DCMAKE_OSX_ARCHITECTURES=$CMakeOSXArchitectures \
-DCMAKE_STRIP=$(which strip) \
-DCMAKE_STRIP=$StripTool \
-DLLVM_DEFAULT_TARGET_TRIPLE=$LLVMDefaultTargetTriple \
-DLLVM_ENABLE_TERMINFO=OFF \
-DLLVM_EXTERNAL_PROJECTS=coredistools \
-DLLVM_EXTERNAL_COREDISTOOLS_SOURCE_DIR=$SourcesDirectory/coredistools \
-DLLVM_HOST_TRIPLE=$LLVMHostTriple \
-DLLVM_INCLUDE_TESTS=OFF \
-DLLVM_TABLEGEN=$(which llvm-tblgen) \
-DLLVM_TABLEGEN=$TblGenTool \
-DLLVM_TARGETS_TO_BUILD=$LLVMTargetsToBuild \
-DLLVM_TOOL_COREDISTOOLS_BUILD=ON \
$SourcesDirectory/llvm-project/llvm
Expand All @@ -103,21 +121,21 @@ else
-G "Unix Makefiles" \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_CROSSCOMPILING=$CMakeCrossCompiling \
-DCMAKE_C_COMPILER=$(which clang) \
-DCMAKE_C_COMPILER=$(command -v clang) \
-DCMAKE_C_FLAGS="-target $LLVMHostTriple --sysroot=$CrossRootfsDirectory" \
-DCMAKE_CXX_COMPILER=$(which clang++) \
-DCMAKE_CXX_COMPILER=$(command -v clang++) \
-DCMAKE_CXX_FLAGS="-target $LLVMHostTriple --sysroot=$CrossRootfsDirectory" \
-DCMAKE_INCLUDE_PATH=$CrossRootfsDirectory/usr/include \
-DCMAKE_INSTALL_PREFIX=$StagingDirectory \
-DCMAKE_LIBRARY_PATH=$CrossRootfsDirectory/usr/lib/$LLVMHostTriple \
-DCMAKE_STRIP=/usr/$LLVMHostTriple/bin/strip \
-DCMAKE_STRIP=$StripTool \
-DLLVM_DEFAULT_TARGET_TRIPLE=$LLVMDefaultTargetTriple \
-DLLVM_ENABLE_TERMINFO=OFF \
-DLLVM_EXTERNAL_PROJECTS=coredistools \
-DLLVM_EXTERNAL_COREDISTOOLS_SOURCE_DIR=$SourcesDirectory/coredistools \
-DLLVM_HOST_TRIPLE=$LLVMHostTriple \
-DLLVM_INCLUDE_TESTS=OFF \
-DLLVM_TABLEGEN=$(which llvm-tblgen) \
-DLLVM_TABLEGEN=$TblGenTool \
-DLLVM_TARGETS_TO_BUILD=$LLVMTargetsToBuild \
-DLLVM_TOOL_COREDISTOOLS_BUILD=ON \
$SourcesDirectory/llvm-project/llvm
Expand Down
116 changes: 116 additions & 0 deletions build-tblgen.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
#!/usr/bin/env bash
#
# Build the Linux/Mac llvm-tblgen tool. Note that this will be run during the
# Linux/Mac coredistools build. So, we only build the versions that we'll use
# during those builds. Thus, we need a linux-x64 version (used for
# linux-x64, linux-arm, linux-arm64 builds) and osx-x64 version (used for
# osx-x64 and osx-arm64 builds).
#
# The linux-x64 build is itself a cross-build, using CBL-Mariner container to build.

TargetOSArchitecture=$1
CrossRootfsDirectory=$2

EnsureCrossRootfsDirectoryExists () {
if [ ! -d "$CrossRootfsDirectory" ]; then
echo "Invalid or unspecified CrossRootfsDirectory: $CrossRootfsDirectory"
exit 1
fi
}

CMakeOSXArchitectures=
LLVMTargetsToBuild="AArch64;ARM;X86"

case "$TargetOSArchitecture" in
linux-x64)
CMakeCrossCompiling=ON
LLVMHostTriple=x86_64-linux-gnu
EnsureCrossRootfsDirectoryExists
;;

linux-loongarch64)
CMakeCrossCompiling=OFF
LLVMHostTriple=loongarch64-linux-gnu
LLVMTargetsToBuild="LoongArch"
;;

osx-x64)
CMakeCrossCompiling=OFF
CMakeOSXArchitectures=x86_64
LLVMHostTriple=x86_64-apple-darwin
;;

*)
echo "Unknown target OS and architecture: $TargetOSArchitecture"
exit 1
esac

RootDirectory="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
SourcesDirectory=$RootDirectory/src
BinariesDirectory=$RootDirectory/obj
StagingDirectory=$RootDirectory/bin

command -v cmake >/dev/null 2>&1

if [ "$?" -ne 0 ]; then
echo "ERROR: cmake is not found in the PATH"
exit 1
fi

if [ ! -d $BinariesDirectory ]; then
mkdir -p $BinariesDirectory
fi

pushd "$BinariesDirectory"

if [ -z "$CrossRootfsDirectory" ]; then
cmake \
-G "Unix Makefiles" \
-DCMAKE_CROSSCOMPILING=$CMakeCrossCompiling \
-DCMAKE_C_COMPILER=$(command -v clang) \
-DCMAKE_CXX_COMPILER=$(command -v clang++) \
-DCMAKE_INSTALL_PREFIX=$RootDirectory \
-DCMAKE_OSX_ARCHITECTURES=$CMakeOSXArchitectures \
-DLLVM_TARGETS_TO_BUILD=$LLVMTargetsToBuild \
$SourcesDirectory/llvm-project/llvm
else
cmake \
-G "Unix Makefiles" \
-DCMAKE_INSTALL_PREFIX=$RootDirectory \
-DCMAKE_CROSSCOMPILING=$CMakeCrossCompiling \
-DCMAKE_C_COMPILER=$(command -v clang) \
-DCMAKE_C_FLAGS="--sysroot=$CrossRootfsDirectory" \
-DCMAKE_CXX_COMPILER=$(command -v clang++) \
-DCMAKE_CXX_FLAGS="--sysroot=$CrossRootfsDirectory" \
-DCMAKE_INCLUDE_PATH=$CrossRootfsDirectory/usr/include \
-DCMAKE_INSTALL_PREFIX=$RootDirectory \
-DCMAKE_LIBRARY_PATH=$CrossRootfsDirectory/usr/lib/$LLVMHostTriple \
-DLLVM_TARGETS_TO_BUILD=$LLVMTargetsToBuild \
$SourcesDirectory/llvm-project/llvm
fi

popd

if [ "$?" -ne 0 ]; then
echo "ERROR: cmake exited with code $1"
exit 1
fi

cmake \
--build $BinariesDirectory \
--target llvm-tblgen \
--config Release

if [ "$?" -ne 0 ]; then
echo "ERROR: cmake exited with code $1"
exit 1
fi

if [ ! -d $StagingDirectory ]; then
mkdir -p $StagingDirectory
fi

# Copy llvm-tblgen from BinariesDirectory to StagingDirectory
find $BinariesDirectory -name llvm-tblgen

exit 0
75 changes: 28 additions & 47 deletions coredistools.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,17 +28,22 @@ trigger:

resources:
containers:
- container: ubuntu-18.04-arm
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-20220312201346-b9de666
- container: ubuntu-18.04-arm64
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm64-20220312201346-b2c2436
- container: linux_x64
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64-20230414190614-8100bf7
- container: linux_arm
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm-20230414190614-8100bf7
- container: linux_arm64
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64-20230414190614-8100bf7

variables:
LLVMRepositoryUri: https://github.com/llvm/llvm-project.git
LLVMSourceBundle: llvm-project.bundle
LLVMSourceVersion: llvmorg-13.0.1
LLVMSourceVersion: llvmorg-16.0.1

jobs:

################################ Check out LLVM source tree; publish to artifacts for future jobs

- job: checkout_llvm
displayName: Checkout LLVM

Expand All @@ -61,6 +66,8 @@ jobs:
artifact: $(LLVMSourceBundle)
displayName: Publish LLVM bundle

################################ Cross-build coredistools for linux-x64, linux-arm, linux-arm64 in CBL-Mariner Docker container

- job: crossbuild_coredistools_linux
dependsOn: checkout_llvm
displayName: Build coredistools Linux
Expand All @@ -72,13 +79,18 @@ jobs:

strategy:
matrix:
x64:
ContainerImage: linux_x64
CrossRootfsDirectory: /crossrootfs/x64
TargetOSArchitecture: linux-x64

arm:
ContainerImage: ubuntu-18.04-arm
ContainerImage: linux_arm
CrossRootfsDirectory: /crossrootfs/arm
TargetOSArchitecture: linux-arm

arm64:
ContainerImage: ubuntu-18.04-arm64
ContainerImage: linux_arm64
CrossRootfsDirectory: /crossrootfs/arm64
TargetOSArchitecture: linux-arm64

Expand All @@ -88,45 +100,14 @@ jobs:
steps:
- template: /eng/download-checkout-llvm.yml

- template: /eng/download-llvm-release.yml
parameters:
os: linux
release: $(LLVMSourceVersion)

- script: ./build-coredistools.sh $(TargetOSArchitecture) $(CrossRootfsDirectory)
displayName: Build coredistools

- publish: $(Build.SourcesDirectory)/artifacts/$(TargetOSArchitecture)/bin/libcoredistools.so
artifact: coredistools-$(TargetOSArchitecture)
displayName: Publish coredistools

- job: build_coredistools_linux_x64
dependsOn: checkout_llvm
displayName: Build coredistools Linux x64

pool:
vmImage: ubuntu-18.04

variables:
TargetOSArchitecture: linux-x64

workspace:
clean: all

steps:
- template: /eng/download-checkout-llvm.yml

- template: /eng/download-llvm-release.yml
parameters:
os: linux
release: $(LLVMSourceVersion)

- script: ./build-coredistools.sh $(TargetOSArchitecture)
displayName: Build coredistools

- publish: $(Build.SourcesDirectory)/artifacts/$(TargetOSArchitecture)/bin/libcoredistools.so
artifact: coredistools-$(TargetOSArchitecture)
displayName: Publish coredistools
################################ Build coredistools for macos-x64, macos-arm64

- job: build_coredistools_macos
dependsOn: checkout_llvm
Expand All @@ -139,29 +120,26 @@ jobs:
matrix:
x64:
TargetOSArchitecture: osx-x64
VMImage: macOS-10.15
VMImage: macOS-latest
arm64:
TargetOSArchitecture: osx-arm64
VMImage: macOS-11
VMImage: macOS-latest

workspace:
clean: all

steps:
- template: /eng/download-checkout-llvm.yml

- template: /eng/download-llvm-release.yml
parameters:
os: macos
release: $(LLVMSourceVersion)

- script: ./build-coredistools.sh $(TargetOSArchitecture)
displayName: Build coredistools

- publish: $(Build.SourcesDirectory)/artifacts/$(TargetOSArchitecture)/bin/libcoredistools.dylib
artifact: coredistools-$(TargetOSArchitecture)
displayName: Publish coredistools

################################ Build llvm-tblgen on Windows

- job: build_tblgen_windows
dependsOn: checkout_llvm
displayName: Build llvm-tblgen Windows
Expand All @@ -182,6 +160,8 @@ jobs:
artifact: tblgen-windows
displayName: Publish llvm-tblgen

################################ Build coredistools for win-x64, win-x86, win-arm64

- job: build_coredistools_windows
dependsOn:
- checkout_llvm
Expand Down Expand Up @@ -220,10 +200,11 @@ jobs:
artifact: coredistools-$(TargetOSArchitecture)
displayName: Publish coredistools

################################ Build coredistools NuGet packages

- job: build_coredistools_nuget_packages
dependsOn:
- crossbuild_coredistools_linux
- build_coredistools_linux_x64
- build_coredistools_macos
- build_coredistools_windows
displayName: Build coredistools NuGet packages
Expand Down
Loading

0 comments on commit d07dbed

Please sign in to comment.