Merge pull request #558 from Chilledheart/build_add_mingw_target

build: add mingw target (for llvm-mingw build only)

.github/workflows/compiler.yml

@@ -206,70 +206,15 @@ jobs:
           sudo apt-get install -y cmake ninja-build nasm zip
           # required by mbedtls build
           sudo apt-get install -y python3-jsonschema python3-jinja2
-      - name: Set toolchain cmake files
-        run: |
-          cat > MinGW.cmake << EOF
-          # Sample toolchain file for building for Windows from an Ubuntu Linux system.
-          #
-          # Typical usage:
-          #    *) install cross compiler: sudo apt-get install mingw-w64
-          #    *) cd build
-          #    *) cmake -DCMAKE_TOOLCHAIN_FILE=~/mingw-w64-x86_64.cmake ..
-
-          set(CMAKE_SYSTEM_NAME Windows)
-          set(CMAKE_SYSTEM_PROCESSOR ${{ matrix.arch }})
-          set(GNU_TARGET ${{ matrix.arch }}-w64-mingw32)
-
-          set(CMAKE_C_COMPILER_TARGET \${GNU_TARGET} CACHE STRING "")
-          set(CMAKE_CXX_COMPILER_TARGET \${GNU_TARGET} CACHE STRING "")
-          set(CMAKE_ASM_COMPILER_TARGET \${GNU_TARGET} CACHE STRING "")
-
-          set(CMAKE_ASM_FLAGS "--rtlib=compiler-rt")
-          set(CMAKE_C_FLAGS "--rtlib=compiler-rt -Wno-unused-command-line-argument")
-          set(CMAKE_CXX_FLAGS "--rtlib=compiler-rt -stdlib=libc++ -Wno-unused-command-line-argument")
-          set(CMAKE_SHARED_LINKER_FLAGS "--rtlib=compiler-rt --unwindlib=libunwind")
-          set(CMAKE_EXE_LINKER_FLAGS "--rtlib=compiler-rt --unwindlib=libunwind")
-
-          # cross compilers to use for C and C++
-          set(CMAKE_C_COMPILER "/opt/llvm-mingw-20230614-ucrt-ubuntu-20.04-x86_64/bin/clang")
-          set(CMAKE_CXX_COMPILER "/opt/llvm-mingw-20230614-ucrt-ubuntu-20.04-x86_64/bin/clang++")
-          set(CMAKE_LINKER "/opt/llvm-mingw-20230614-ucrt-ubuntu-20.04-x86_64/bin/ld.lld")
-          set(CMAKE_AR "/opt/llvm-mingw-20230614-ucrt-ubuntu-20.04-x86_64/bin/llvm-ar")
-          set(CMAKE_CXX_COMPILER_AR "/opt/llvm-mingw-20230614-ucrt-ubuntu-20.04-x86_64/bin/llvm-ar")
-          set(CMAKE_RANLIB "/opt/llvm-mingw-20230614-ucrt-ubuntu-20.04-x86_64/bin/llvm-ranlib")
-          set(CMAKE_CXX_COMPILER_RANLIB "/opt/llvm-mingw-20230614-ucrt-ubuntu-20.04-x86_64/bin/llvm-ranlib")
-          # disabled because it is missing from binary
-          # set(CMAKE_RC_COMPILER "/opt/llvm-mingw-20230614-ucrt-ubuntu-20.04-x86_64/bin/llvm-rc")
-          set(CMAKE_RC_COMPILER "/opt/llvm-mingw-20230614-ucrt-ubuntu-20.04-x86_64/bin/\${GNU_TARGET}-windres")
-
-          set(CMAKE_SYSROOT "/opt/llvm-mingw-20230614-ucrt-ubuntu-20.04-x86_64/\${GNU_TARGET}" CACHE STRING "")
-
-          # target environment on the build host system
-          set(CMAKE_FIND_ROOT_PATH /opt/llvm-mingw-20230614-ucrt-ubuntu-20.04-x86_64/\${GNU_TARGET})
-
-          # modify default behavior of FIND_XXX() commands
-          set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
-          set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
-          set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
-          EOF
-          cat > Native.cmake << EOF
-          set(CMAKE_C_COMPILER "${{ github.workspace }}/third_party/llvm-build/Release+Asserts/bin/clang")
-          set(CMAKE_CXX_COMPILER "${{ github.workspace }}/third_party/llvm-build/Release+Asserts/bin/clang++")
-          EOF
       - name: Build
         run: |
-          mkdir build-mingw
-          cd build-mingw
-          cmake -G Ninja -DGUI=on -DBUILD_TESTS=on -DUSE_ICU=off .. \
-            -DCMAKE_TOOLCHAIN_FILE="$PWD/../MinGW.cmake" -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
-            -DCROSS_TOOLCHAIN_FLAGS_TOOLCHAIN_FILE=\"$PWD/../Native.cmake\" \
-            ${{ env.CMAKE_OPTIONS }}
-          ninja yass yass_test
+          ./tools/build --variant gui --arch ${{ matrix.arch }} --system mingw -build-test -cmake-build-type ${{ matrix.build_type }} \
+            --clang-path /opt/llvm-mingw-20230614-ucrt-ubuntu-20.04-x86_64 -nc
       - name: Populate depedencies (Tests-i686)
         if: ${{ matrix.arch == 'i686' }}
         run: |
           # copy curl dll required by yass_test
-          cp -v third_party/curl-8.4.0_7-win32-mingw/bin/libcurl.dll build-mingw
+          cp -v third_party/curl-8.4.0_7-win32-mingw/bin/libcurl.dll build-mingw-${{ matrix.arch }}/
           # fix for missing i386 deb packages from deb.sury.org repository
           sudo add-apt-repository ppa:ondrej/php
           # alternative fix is to downgrade overloaded libpcre2 versions with five locally installed packages which are PHP-related and came from deb.sury.org repository.
@@ -279,12 +224,12 @@ jobs:
         if: ${{ matrix.arch == 'x86_64' }}
         run: |
           # copy curl dll required by yass_test
-          cp -v third_party/curl-8.4.0_7-win64-mingw/bin/libcurl-x64.dll build-mingw
+          cp -v third_party/curl-8.4.0_7-win64-mingw/bin/libcurl-x64.dll build-mingw-${{ matrix.arch }}/
           sudo apt-get update -qq && sudo apt-get install -y wine wine64
       - name: Run tests
         if: ${{ matrix.arch == 'i686' || matrix.arch == 'x86_64' }}
         run: |
-          cd build-mingw
+          cd build-mingw-${{ matrix.arch }}
           wine yass_test.exe -logtostderr -v 1 --gtest_shuffle --gtest_repeat=25 --gtest_throw_on_failure
           wine yass_test.exe -logtostderr -v 1 --gtest_shuffle --gtest_repeat=25 --gtest_throw_on_failure --ipv6_mode
 

CMakeLists.txt

@@ -1631,10 +1631,11 @@ endif()
 if (WIN32)
   # FIXME for mingw, CMAKE_SYSTEM_PROCESSOR can't tell which target it is building,
   # it needs to exclude arm64 build correctly.
-  if (MINGW OR MSVC_PROCESSOR_ARCHITECTURE STREQUAL "x86" OR MSVC_PROCESSOR_ARCHITECTURE STREQUAL "amd64")
+  if ((MINGW AND NOT OS_AARCH64) OR MSVC_PROCESSOR_ARCHITECTURE STREQUAL "x86" OR MSVC_PROCESSOR_ARCHITECTURE STREQUAL "amd64")
     message(STATUS "Built with sse3 support")
     set(CMAKE_C_FLAGS   "${CMAKE_C_FLAGS} -msse3")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse3")
+    list(APPEND YASS_APP_FEATURES "sse3")
   endif()
 endif()
 

cmake/platforms/MinGW.cmake

@@ -0,0 +1,37 @@
+set(CMAKE_SYSTEM_NAME Windows)
+set(CMAKE_SYSTEM_PROCESSOR ${GCC_SYSTEM_PROCESSOR})
+
+set(CMAKE_C_COMPILER_TARGET ${GCC_TARGET} CACHE STRING "")
+set(CMAKE_CXX_COMPILER_TARGET ${GCC_TARGET} CACHE STRING "")
+set(CMAKE_ASM_COMPILER_TARGET ${GCC_TARGET} CACHE STRING "")
+
+set(CMAKE_ASM_FLAGS "--start-no-unused-arguments -rtlib=compiler-rt -fuse-ld=lld --end-no-unused-arguments")
+set(CMAKE_C_FLAGS "--start-no-unused-arguments -rtlib=compiler-rt -fuse-ld=lld --end-no-unused-arguments")
+set(CMAKE_CXX_FLAGS "--start-no-unused-arguments -rtlib=compiler-rt -stdlib=libc++ -fuse-ld=lld --end-no-unused-arguments")
+set(CMAKE_SHARED_LINKER_FLAGS "-rtlib=compiler-rt -unwindlib=libunwind -fuse-ld=lld")
+set(CMAKE_EXE_LINKER_FLAGS "-rtlib=compiler-rt -unwindlib=libunwind -fuse-ld=lld")
+
+# cross compilers to use for C and C++
+set(CMAKE_C_COMPILER "${LLVM_SYSROOT}/bin/clang")
+set(CMAKE_CXX_COMPILER "${LLVM_SYSROOT}/bin/clang++")
+set(CMAKE_AR "${LLVM_SYSROOT}/bin/llvm-ar")
+set(CMAKE_ASM_COMPILER_AR "${LLVM_SYSROOT}/bin/llvm-ar")
+set(CMAKE_C_COMPILER_AR "${LLVM_SYSROOT}/bin/llvm-ar")
+set(CMAKE_CXX_COMPILER_AR "${LLVM_SYSROOT}/bin/llvm-ar")
+set(CMAKE_RANLIB "${LLVM_SYSROOT}/bin/llvm-ranlib")
+set(CMAKE_ASM_COMPILER_RANLIB "${LLVM_SYSROOT}/bin/llvm-ranlib")
+set(CMAKE_C_COMPILER_RANLIB "${LLVM_SYSROOT}/bin/llvm-ranlib")
+set(CMAKE_CXX_COMPILER_RANLIB "${LLVM_SYSROOT}/bin/llvm-ranlib")
+# disabled because it is missing from binary
+# set(CMAKE_RC_COMPILER "${LLVM_SYSROOT}/bin/llvm-rc")
+set(CMAKE_RC_COMPILER "${LLVM_SYSROOT}/bin/${GCC_TARGET}-windres")
+
+set(CMAKE_SYSROOT "${LLVM_SYSROOT}/${GCC_TARGET}" CACHE STRING "")
+
+# target environment on the build host system
+set(CMAKE_FIND_ROOT_PATH ${LLVM_SYSROOT}/${GCC_TARGET})
+
+# modify default behavior of FIND_XXX() commands
+set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
+set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
+set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)

tools/build.go

@@ -87,6 +87,8 @@ func getAppName() string {
 		return APPNAME + ".exe"
 	} else if systemNameFlag == "darwin" {
 		return APPNAME + ".app"
+	} else if systemNameFlag == "mingw" {
+		return APPNAME + ".exe"
 	} else if systemNameFlag == "android" {
 		if APPNAME == "yass" {
 			return "lib" + APPNAME + ".so"
@@ -336,6 +338,18 @@ func getLLVMTargetTripleMSVC(msvcTargetArch string) string {
 	return ""
 }
 
+func getMinGWTargetAndAppAbi(arch string) (string, string) {
+	if arch == "x86" || arch == "i686" {
+		return "i686-w64-mingw32", "i686"
+	} else if arch == "x64" || arch == "x86_64" || arch == "amd64" {
+		return "x86_64-w64-mingw32", "x86_64"
+	} else if arch == "arm64" || arch == "aarch64" {
+		return "aarch64-w64-mingw32", "aarch64"
+	}
+	glog.Fatalf("Invalid MinGW arch: %s", arch)
+	return "", ""
+}
+
 // build/config/android/abi.gni
 func getAndroidTargetAndAppAbi(arch string) (string, string) {
 	if arch == "x64" {
@@ -770,6 +784,26 @@ func buildStageGenerateBuildScript() {
 		}
 	}
 
+	if systemNameFlag == "mingw" {
+		glog.Infof("Using llvm-mingw dir %s", clangPath);
+		cmakeArgs = append(cmakeArgs, fmt.Sprintf("-DCROSS_TOOLCHAIN_FLAGS_TOOLCHAIN_FILE=%s/Native.cmake", buildDir))
+
+		ccCompiler := os.Getenv("CC")
+		cxxCompiler := os.Getenv("CXX")
+		nativeToolChainContent := strings.Replace(fmt.Sprintf("set(CMAKE_C_COMPILER \"%s\")\n", ccCompiler), "\\", "/", -1)
+		nativeToolChainContent += strings.Replace(fmt.Sprintf("set(CMAKE_CXX_COMPILER \"%s\")\n", cxxCompiler), "\\", "/", -1)
+		err := ioutil.WriteFile("Native.cmake", []byte(nativeToolChainContent), 0666)
+		if err != nil {
+			glog.Fatalf("%v", err)
+		}
+
+		targetTriple, targetAbi := getMinGWTargetAndAppAbi(archFlag)
+		cmakeArgs = append(cmakeArgs, fmt.Sprintf("-DCMAKE_TOOLCHAIN_FILE=%s/../cmake/platforms/MinGW.cmake", buildDir))
+		cmakeArgs = append(cmakeArgs, fmt.Sprintf("-DLLVM_SYSROOT=%s", clangPath))
+		cmakeArgs = append(cmakeArgs, fmt.Sprintf("-DGCC_SYSTEM_PROCESSOR=%s", targetAbi))
+		cmakeArgs = append(cmakeArgs, fmt.Sprintf("-DGCC_TARGET=%s", targetTriple))
+	}
+
 	if systemNameFlag == "ios" {
 		cmakeArgs = append(cmakeArgs, fmt.Sprintf("-DCMAKE_TOOLCHAIN_FILE=%s/../cmake/platforms/ios.toolchain.cmake", buildDir))
 		cmakeArgs = append(cmakeArgs, fmt.Sprintf("-DDEPLOYMENT_TARGET=%s", iosVersionMinFlag))
@@ -956,7 +990,7 @@ func postStateStripBinaries() {
 	if systemNameFlag == "windows" {
 		return
 	}
-	if systemNameFlag == "harmony" || systemNameFlag == "android" || systemNameFlag == "linux" || systemNameFlag == "freebsd" {
+	if systemNameFlag == "mingw" || systemNameFlag == "android" || systemNameFlag == "harmony" || systemNameFlag == "linux" || systemNameFlag == "freebsd" {
 		objcopy := filepath.Join(clangPath, "bin", "llvm-objcopy")
 		if _, err := os.Stat(objcopy); errors.Is(err, os.ErrNotExist) {
 			objcopy = "objcopy"
@@ -1470,7 +1504,7 @@ func postStateArchives() map[string][]string {
 
 	ext := ".zip"
 
-	if systemNameFlag == "harmony" || systemNameFlag == "android" || systemNameFlag == "linux" || systemNameFlag == "freebsd" {
+	if systemNameFlag == "android" || systemNameFlag == "harmony" || systemNameFlag == "linux" || systemNameFlag == "freebsd" {
 		ext = ".tgz"
 	}
 
@@ -1547,7 +1581,7 @@ func postStateArchives() map[string][]string {
 	if systemNameFlag == "windows" {
 		archiveFiles(debugArchive, archivePrefix, []string{APPNAME + ".pdb"})
 		dbgPaths = append(dbgPaths, APPNAME+".pdb")
-	} else if systemNameFlag == "harmony" || systemNameFlag == "android" || systemNameFlag == "linux" || systemNameFlag == "freebsd" {
+	} else if systemNameFlag == "mingw" || systemNameFlag == "android" || systemNameFlag == "harmony" || systemNameFlag == "linux" || systemNameFlag == "freebsd" {
 		archiveFiles(debugArchive, archivePrefix, []string{getAppName() + ".dbg"})
 		dbgPaths = append(dbgPaths, APPNAME+".dbg")
 	} else if systemNameFlag == "darwin" {