Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove bash dependency from init-compiler #77304

Merged
merged 12 commits into from
Dec 21, 2022
81 changes: 37 additions & 44 deletions eng/common/native/init-compiler.sh
Original file line number Diff line number Diff line change
@@ -1,30 +1,25 @@
#!/usr/bin/env bash
#!/bin/sh
#
# This file detects the C/C++ compiler and exports it to the CC/CXX environment variables
#
# NOTE: some scripts source this file and rely on stdout being empty, make sure to not output anything here!

if [[ "$#" -lt 3 ]]; then
if [ -z "$build_arch" ] || [ -z "$compiler" ]; then
echo "Usage..."
echo "init-compiler.sh <script directory> <Architecture> <compiler>"
echo "Specify the script directory."
echo "build_arch=<ARCH> compiler=<NAME> init-compiler.sh"
echo "Specify the target architecture."
echo "Specify the name of compiler (clang or gcc)."
exit 1
fi

nativescriptroot="$1"
build_arch="$2"
compiler="$3"

case "$compiler" in
clang*|-clang*|--clang*)
# clangx.y or clang-x.y
version="$(echo "$compiler" | tr -d '[:alpha:]-=')"
parts=(${version//./ })
majorVersion="${parts[0]}"
minorVersion="${parts[1]}"
if [[ -z "$minorVersion" && "$majorVersion" -le 6 ]]; then
majorVersion="${version%%.*}"
[ -z "${version##*.*}" ] && minorVersion="${version#*.}"

if [ -z "$minorVersion" ] && [ -n "$majorVersion" ] && [ "$majorVersion" -le 6 ]; then
minorVersion=0;
fi
compiler=clang
Expand All @@ -33,23 +28,20 @@ case "$compiler" in
gcc*|-gcc*|--gcc*)
# gccx.y or gcc-x.y
version="$(echo "$compiler" | tr -d '[:alpha:]-=')"
parts=(${version//./ })
majorVersion="${parts[0]}"
minorVersion="${parts[1]}"
majorVersion="${version%%.*}"
[ -z "${version##*.*}" ] && minorVersion="${version#*.}"
compiler=gcc
;;
esac

cxxCompiler="$compiler++"

. "$nativescriptroot"/../pipeline-logging-functions.sh

# clear the existing CC and CXX from environment
CC=
CXX=
LDFLAGS=

if [[ "$compiler" == "gcc" ]]; then cxxCompiler="g++"; fi
if [ "$compiler" = "gcc" ]; then cxxCompiler="g++"; fi

check_version_exists() {
desired_version=-1
Expand All @@ -66,74 +58,75 @@ check_version_exists() {
echo "$desired_version"
}

if [[ -z "$CLR_CC" ]]; then
if [ -z "$CLR_CC" ]; then

# Set default versions
if [[ -z "$majorVersion" ]]; then
if [ -z "$majorVersion" ]; then
# note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero.
if [[ "$compiler" == "clang" ]]; then versions=( 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5 )
elif [[ "$compiler" == "gcc" ]]; then versions=( 12 11 10 9 8 7 6 5 4.9 ); fi

for version in "${versions[@]}"; do
parts=(${version//./ })
desired_version="$(check_version_exists "${parts[0]}" "${parts[1]}")"
if [[ "$desired_version" != "-1" ]]; then majorVersion="${parts[0]}"; break; fi
if [ "$compiler" = "clang" ]; then versions="15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5"
elif [ "$compiler" = "gcc" ]; then versions="12 11 10 9 8 7 6 5 4.9"; fi

for version in $versions; do
_major="${version%%.*}"
[ -z "${version##*.*}" ] && _minor="${version#*.}"
desired_version="$(check_version_exists "$_major" "$_minor")"
if [ "$desired_version" != "-1" ]; then majorVersion="$_major"; break; fi
done

if [[ -z "$majorVersion" ]]; then
if [ -z "$majorVersion" ]; then
if command -v "$compiler" > /dev/null; then
if [[ "$(uname)" != "Darwin" ]]; then
Write-PipelineTelemetryError -category "Build" -type "warning" "Specific version of $compiler not found, falling back to use the one in PATH."
if [ "$(uname)" != "Darwin" ]; then
echo "Warning: Specific version of $compiler not found, falling back to use the one in PATH."
fi
CC="$(command -v "$compiler")"
CXX="$(command -v "$cxxCompiler")"
else
Write-PipelineTelemetryError -category "Build" "No usable version of $compiler found."
echo "No usable version of $compiler found."
exit 1
fi
else
if [[ "$compiler" == "clang" && "$majorVersion" -lt 5 ]]; then
if [[ "$build_arch" == "arm" || "$build_arch" == "armel" ]]; then
if [ "$compiler" = "clang" ] && [ "$majorVersion" -lt 5 ]; then
if [ "$build_arch" = "arm" ] || [ "$build_arch" = "armel" ]; then
if command -v "$compiler" > /dev/null; then
Write-PipelineTelemetryError -category "Build" -type "warning" "Found clang version $majorVersion which is not supported on arm/armel architectures, falling back to use clang from PATH."
echo "Warning: Found clang version $majorVersion which is not supported on arm/armel architectures, falling back to use clang from PATH."
CC="$(command -v "$compiler")"
CXX="$(command -v "$cxxCompiler")"
else
Write-PipelineTelemetryError -category "Build" "Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH."
echo "Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH."
exit 1
fi
fi
fi
fi
else
desired_version="$(check_version_exists "$majorVersion" "$minorVersion")"
if [[ "$desired_version" == "-1" ]]; then
Write-PipelineTelemetryError -category "Build" "Could not find specific version of $compiler: $majorVersion $minorVersion."
if [ "$desired_version" = "-1" ]; then
echo "Could not find specific version of $compiler: $majorVersion $minorVersion."
exit 1
fi
fi

if [[ -z "$CC" ]]; then
if [ -z "$CC" ]; then
CC="$(command -v "$compiler$desired_version")"
CXX="$(command -v "$cxxCompiler$desired_version")"
if [[ -z "$CXX" ]]; then CXX="$(command -v "$cxxCompiler")"; fi
if [ -z "$CXX" ]; then CXX="$(command -v "$cxxCompiler")"; fi
fi
else
if [[ ! -f "$CLR_CC" ]]; then
Write-PipelineTelemetryError -category "Build" "CLR_CC is set but path '$CLR_CC' does not exist"
if [ ! -f "$CLR_CC" ]; then
echo "CLR_CC is set but path '$CLR_CC' does not exist"
exit 1
fi
CC="$CLR_CC"
CXX="$CLR_CXX"
fi

if [[ -z "$CC" ]]; then
Write-PipelineTelemetryError -category "Build" "Unable to find $compiler."
if [ -z "$CC" ]; then
echo "Unable to find $compiler."
exit 1
fi

# Only lld version >= 9 can be considered stable. lld doesn't support s390x.
if [[ "$compiler" == "clang" && "$majorVersion" -ge 9 && "$build_arch" != "s390x" ]]; then
if [ "$compiler" = "clang" ] && [ -n "$majorVersion" ] && [ "$majorVersion" -ge 9 ] && [ "$build_arch" != "s390x" ]; then
if "$CC" -fuse-ld=lld -Wl,--version >/dev/null 2>&1; then
LDFLAGS="-fuse-ld=lld"
fi
Expand Down
2 changes: 1 addition & 1 deletion eng/native/gen-buildsys.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ compiler="$5"

if [[ "$compiler" != "default" ]]; then
nativescriptroot="$( cd -P "$scriptroot/../common/native" && pwd )"
source "$nativescriptroot/init-compiler.sh" "$nativescriptroot" "$host_arch" "$compiler"
build_arch="$host_arch" compiler="$compiler" . "$nativescriptroot/init-compiler.sh"

CCC_CC="$CC"
CCC_CXX="$CXX"
Expand Down
24 changes: 22 additions & 2 deletions eng/testing/tests.singlefile.targets
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,6 @@
<PropertyGroup Condition="'$(TestNativeAot)' == 'true'">
<IlcToolsPath>$(CoreCLRILCompilerDir)</IlcToolsPath>
<IlcToolsPath Condition="'$(TargetArchitecture)' != '$(BuildArchitecture)'">$(CoreCLRCrossILCompilerDir)</IlcToolsPath>
<CppCompilerAndLinker Condition="'$(TargetArchitecture)' != '$(BuildArchitecture)' and '$(HostOS)' == 'linux' and '$(RuntimeIdentifier)' == 'linux-musl-arm64'">clang-15</CppCompilerAndLinker>
<CppCompilerAndLinker Condition="'$(TargetArchitecture)' != '$(BuildArchitecture)' and '$(HostOS)' == 'linux' and '$(RuntimeIdentifier)' != 'linux-musl-arm64'">clang-9</CppCompilerAndLinker>
<SysRoot Condition="'$(TargetArchitecture)' != '$(BuildArchitecture)' and '$(HostOS)' != 'windows'">$(ROOTFS_DIR)</SysRoot>
<IlcBuildTasksPath>$(CoreCLRILCompilerDir)netstandard/ILCompiler.Build.Tasks.dll</IlcBuildTasksPath>
<IlcSdkPath>$(CoreCLRAotSdkDir)</IlcSdkPath>
Expand Down Expand Up @@ -84,6 +82,28 @@
</ItemGroup>
</Target>

<!--
Use init-compiler.sh to locate the compiler toolchain which was resolved for rest of the build.
This target is essentially an override hook which is called before `SetupOSSpecificProps` in
`Microsoft.NETCore.Native.Unix.targets`. Note that the input is `CppCompilerAndLinker`
and the output is `CppLinker`, because from `SetupOSSpecificProps` onwards, we only use `CppLinker`
(when alternative compiler, i.e. gcc, is not selected)
-->
<Target Name="LocateNativeCompiler"
am11 marked this conversation as resolved.
Show resolved Hide resolved
Condition="'$(TestNativeAot)' == 'true' and '$(HostOS)' != 'windows'"
BeforeTargets="SetupOSSpecificProps">
<PropertyGroup>
<CppCompilerAndLinker Condition="'$(CppCompilerAndLinker)' == ''">clang</CppCompilerAndLinker>
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

looking at the whole PR again, maybe it'd be better to not use this property name at all and go with what we use in the NativeExports.csproj:

    <PropertyGroup>
      <NativeCompiler>$(Compiler)</NativeCompiler>
      <NativeCompiler Condition="'$(NativeCompiler)' == ''">clang</NativeCompiler>
    </PropertyGroup>

that'd also support overriding via the $(Compiler) property.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Compiler == CppCompilerAndLinker here, see eng/build.sh

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

omg, so many abstractions 😄

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

:)

I am all for replacing CppCompilerAndLinker and CppLinker in favor of NativeCompiler as property name, but I think some consumers are relying on CppCompilerAndLinker.

</PropertyGroup>

<Exec Command="sh -c 'build_arch=&quot;$(TargetArchitecture)&quot; compiler=&quot;$(CppCompilerAndLinker)&quot; . &quot;$(RepositoryEngineeringDir)/common/native/init-compiler.sh&quot; &amp;&amp; echo $CC' 2>/dev/null"
EchoOff="true"
ConsoleToMsBuild="true"
StandardOutputImportance="Low">
<Output TaskParameter="ConsoleOutput" PropertyName="CppLinker" />
akoeplinger marked this conversation as resolved.
Show resolved Hide resolved
</Exec>
</Target>

<Target Name="PublishTestAsSingleFile"
Condition="'$(IsCrossTargetingBuild)' != 'true'"
AfterTargets="Build"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -140,10 +140,10 @@ The .NET Foundation licenses this file to you under the MIT license.
</PropertyGroup>

<PropertyGroup Condition="'$(ObjCopyName)' == '' and '$(TargetOS)' != 'OSX'">
<ObjCopyName Condition="'$(CppCompilerAndLinker)' != 'clang'">objcopy</ObjCopyName>
<ObjCopyName Condition="'$(CppCompilerAndLinker)' == 'clang'">llvm-objcopy</ObjCopyName>
<ObjCopyName Condition="!$(CppCompilerAndLinker.Contains('clang'))">objcopy</ObjCopyName>
<ObjCopyName Condition="$(CppCompilerAndLinker.Contains('clang'))">llvm-objcopy</ObjCopyName>
<ObjCopyNameAlternative />
<ObjCopyNameAlternative Condition="'$(CppCompilerAndLinker)' == 'clang'">objcopy</ObjCopyNameAlternative>
<ObjCopyNameAlternative Condition="$(CppCompilerAndLinker.Contains('clang'))">objcopy</ObjCopyNameAlternative>
</PropertyGroup>

<Error Condition="'$(_WhereLinker)' != '0' and '$(TargetOS)' == 'OSX'" Text="Platform linker ('$(CppLinker)') not found in PATH. Try installing Xcode to resolve the problem." />
Expand Down
17 changes: 15 additions & 2 deletions src/coreclr/tools/aot/crossgen2/crossgen2.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,6 @@
<PropertyGroup Condition="'$(NativeAotSupported)' == 'true'">
<IlcToolsPath>$(CoreCLRILCompilerDir)</IlcToolsPath>
<IlcToolsPath Condition="'$(TargetArchitecture)' != '$(BuildArchitecture)'">$(CoreCLRCrossILCompilerDir)</IlcToolsPath>
<CppCompilerAndLinker Condition="'$(TargetArchitecture)' != '$(BuildArchitecture)' and '$(HostOS)' == 'linux' and '$(RuntimeIdentifier)' == 'linux-musl-arm64'">clang-15</CppCompilerAndLinker>
<CppCompilerAndLinker Condition="'$(TargetArchitecture)' != '$(BuildArchitecture)' and '$(HostOS)' == 'linux' and '$(RuntimeIdentifier)' != 'linux-musl-arm64'">clang-9</CppCompilerAndLinker>
<SysRoot Condition="'$(TargetArchitecture)' != '$(BuildArchitecture)' and '$(HostOS)' != 'windows'">$(ROOTFS_DIR)</SysRoot>
<IlcBuildTasksPath>$(CoreCLRILCompilerDir)netstandard/ILCompiler.Build.Tasks.dll</IlcBuildTasksPath>
<IlcSdkPath>$(CoreCLRAotSdkDir)</IlcSdkPath>
Expand All @@ -59,4 +57,19 @@
</ItemGroup>
</Target>

<Target Name="LocateNativeCompiler"
Condition="'$(NativeAotSupported)' == 'true' and '$(_IsPublishing)' == 'true' and '$(HostOS)' != 'windows'"
BeforeTargets="SetupOSSpecificProps">
<PropertyGroup>
<CppCompilerAndLinker Condition="'$(CppCompilerAndLinker)' == ''">clang</CppCompilerAndLinker>
</PropertyGroup>

<Exec Command="sh -c 'build_arch=&quot;$(TargetArchitecture)&quot; compiler=&quot;$(CppCompilerAndLinker)&quot; . &quot;$(RepositoryEngineeringDir)/common/native/init-compiler.sh&quot; &amp;&amp; echo $CC' 2>/dev/null"
akoeplinger marked this conversation as resolved.
Show resolved Hide resolved
EchoOff="true"
ConsoleToMsBuild="true"
StandardOutputImportance="Low">
<Output TaskParameter="ConsoleOutput" PropertyName="CppLinker" />
</Exec>
</Target>

</Project>
Original file line number Diff line number Diff line change
Expand Up @@ -44,14 +44,14 @@
<NativeCompiler Condition="'$(NativeCompiler)' == ''">clang</NativeCompiler>
</PropertyGroup>

<Exec Command="bash -c 'source &quot;$(RepositoryEngineeringDir)/common/native/init-compiler.sh&quot; &quot;$(RepositoryEngineeringDir)/common/native&quot; $(TargetArchitecture) $(NativeCompiler) &amp;&amp; echo $CC' 2>/dev/null"
<Exec Command="sh -c 'build_arch=&quot;$(TargetArchitecture)&quot; compiler=&quot;$(NativeCompiler)&quot; . &quot;$(RepositoryEngineeringDir)/common/native/init-compiler.sh&quot; &amp;&amp; echo $CC' 2>/dev/null"
EchoOff="true"
ConsoleToMsBuild="true"
StandardOutputImportance="Low">
<Output TaskParameter="ConsoleOutput" PropertyName="DnneCompilerCommand" />
</Exec>

<Exec Command="bash -c 'source &quot;$(RepositoryEngineeringDir)/common/native/init-compiler.sh&quot; &quot;$(RepositoryEngineeringDir)/common/native&quot; $(TargetArchitecture) $(NativeCompiler) &amp;&amp; echo $LDFLAGS' 2>/dev/null"
<Exec Command="sh -c 'build_arch=&quot;$(TargetArchitecture)&quot; compiler=&quot;$(NativeCompiler)&quot; . &quot;$(RepositoryEngineeringDir)/common/native/init-compiler.sh&quot; &amp;&amp; echo $LDFLAGS' 2>/dev/null"
EchoOff="true"
ConsoleToMsBuild="true"
StandardOutputImportance="Low">
Expand Down
2 changes: 1 addition & 1 deletion src/mono/mono.proj
Original file line number Diff line number Diff line change
Expand Up @@ -513,7 +513,7 @@
<PropertyGroup>
<EMSDK_PATH>$([MSBuild]::EnsureTrailingSlash('$(EMSDK_PATH)'))</EMSDK_PATH>
<_MonoCMakeConfigureCommand>cmake @(_MonoCMakeArgs, ' ') $(MonoCMakeExtraArgs) &quot;$(MonoProjectRoot.TrimEnd('\/'))&quot;</_MonoCMakeConfigureCommand>
<_MonoCMakeConfigureCommand Condition="'$(TargetsBrowser)' != 'true' and '$(_MonoRunInitCompiler)' != 'false' and '$(HostOS)' != 'windows'">bash -c 'source $(RepositoryEngineeringCommonDir)native/init-compiler.sh &quot;$(RepositoryEngineeringCommonDir)native&quot; &quot;$(_CompilerTargetArch)&quot; &quot;$(MonoCCompiler)&quot; &amp;&amp; @(_MonoBuildEnv, ' ') $(_MonoCMakeConfigureCommand)'</_MonoCMakeConfigureCommand>
<_MonoCMakeConfigureCommand Condition="'$(TargetsBrowser)' != 'true' and '$(_MonoRunInitCompiler)' != 'false' and '$(HostOS)' != 'windows'">sh -c 'build_arch=&quot;$(_CompilerTargetArch)&quot; compiler=&quot;$(MonoCCompiler)&quot; . &quot;$(RepositoryEngineeringCommonDir)native/init-compiler.sh&quot; &amp;&amp; @(_MonoBuildEnv, ' ') $(_MonoCMakeConfigureCommand)'</_MonoCMakeConfigureCommand>
<_MonoCMakeConfigureCommand Condition="'$(TargetsBrowser)' != 'true' and '$(_MonoRunInitCompiler)' != 'false' and '$(HostOS)' == 'windows'">call &quot;$(RepositoryEngineeringDir)native\init-vs-env.cmd&quot; $(_CompilerTargetArch) &amp;&amp; cd /D &quot;$(MonoObjDir)&quot; &amp;&amp; @(_MonoBuildEnv, ' ') $(_MonoCMakeConfigureCommand)</_MonoCMakeConfigureCommand>
<_MonoCMakeConfigureCommand Condition="'$(TargetsBrowser)' != 'true' and '$(_MonoRunInitCompiler)' == 'false'">$(_MonoCCOption) $(_MonoCXXOption) @(_MonoBuildEnv, ' ') $(_MonoCMakeConfigureCommand)</_MonoCMakeConfigureCommand>
<_MonoCMakeConfigureCommand Condition="'$(TargetsBrowser)' == 'true' and '$(HostOS)' != 'windows'">bash -c 'source $(EMSDK_PATH)/emsdk_env.sh 2>&amp;1 &amp;&amp; emcmake $(_MonoCMakeConfigureCommand)'</_MonoCMakeConfigureCommand>
Expand Down
27 changes: 17 additions & 10 deletions src/native/libs/System.Globalization.Native/local_build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -10,17 +10,24 @@

# Currently, only Linux is supported

SHIM_SOURCE_DIR=$1/native/src
INTERMEDIATE_OUTPUT_PATH=$2
SHIM_SOURCE_DIR="$1"/native/src
INTERMEDIATE_OUTPUT_PATH="$2"

if [ -d "$SHIM_SOURCE_DIR" ]; then
LOCAL_SHIM_DIR="$INTERMEDIATE_OUTPUT_PATH"/libs/System.Globalization.Native/build
mkdir -p "$LOCAL_SHIM_DIR" && cd "$LOCAL_SHIM_DIR"
if [ $? -ne 0 ]; then echo "local_build.sh::ERROR: Cannot use local build directory"; exit 1; fi
cmake -S "$SHIM_SOURCE_DIR/libs/System.Globalization.Native/" -DLOCAL_BUILD:STRING=1 -DCLR_CMAKE_TARGET_UNIX:STRING=1
if [ $? -ne 0 ]; then echo "local_build.sh::ERROR: cmake failed"; exit 1; fi
make -j
if [ $? -ne 0 ]; then echo "local_build.sh::ERROR: Build failed"; exit 1; fi
fi

exit 0
if ! mkdir -p "$LOCAL_SHIM_DIR" && cd "$LOCAL_SHIM_DIR"; then
echo "local_build.sh::ERROR: Cannot use local build directory"
exit 1
fi

if ! cmake -S "$SHIM_SOURCE_DIR/libs/System.Globalization.Native/" -DLOCAL_BUILD:STRING=1 -DCLR_CMAKE_TARGET_UNIX:STRING=1; then
echo "local_build.sh::ERROR: cmake failed"
exit 1
fi

if ! make -j; then
echo "local_build.sh::ERROR: Build failed"
exit 1
fi
fi