diff --git a/.github/workflows/jvm_tests.yml b/.github/workflows/jvm_tests.yml index 640e41bdbbfe..a2b8b1e6d214 100644 --- a/.github/workflows/jvm_tests.yml +++ b/.github/workflows/jvm_tests.yml @@ -23,6 +23,11 @@ jobs: with: submodules: 'true' + - uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9 # v4.2.1 + with: + distribution: 'temurin' + java-version: '8' + - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4 with: miniforge-variant: Mambaforge @@ -38,19 +43,18 @@ jobs: key: ${{ runner.os }}-m2-${{ hashFiles('./jvm-packages/pom.xml') }} restore-keys: ${{ runner.os }}-m2-${{ hashFiles('./jvm-packages/pom.xml') }} - - name: Build xgboost4j.dll - run: | - mkdir build - cd build - cmake .. -G"Visual Studio 17 2022" -A x64 -DJVM_BINDINGS=ON - cmake --build . --config Release - if: matrix.os == 'windows-latest' - - name: Test XGBoost4J (Core) run: | cd jvm-packages mvn test -B -pl :xgboost4j_2.12 + - name: Test XGBoost4J (Core, Spark, Examples) + run: | + rm -rfv build/ + cd jvm-packages + mvn -B test + if: matrix.os == 'ubuntu-latest' # Distributed training doesn't work on Windows + - name: Extract branch name shell: bash run: | @@ -87,22 +91,9 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_IAM_S3_UPLOADER }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_IAM_S3_UPLOADER }} - - - name: Test XGBoost4J (Core, Spark, Examples) - run: | - rm -rfv build/ - cd jvm-packages - mvn -B test - if: matrix.os == 'ubuntu-latest' # Distributed training doesn't work on Windows - env: - RABIT_MOCK: ON - - - name: Build and Test XGBoost4J with scala 2.13 run: | rm -rfv build/ cd jvm-packages mvn -B clean install test -Pdefault,scala-2.13 if: matrix.os == 'ubuntu-latest' # Distributed training doesn't work on Windows - env: - RABIT_MOCK: ON diff --git a/CMakeLists.txt b/CMakeLists.txt index c69b0d2a3dc7..2eb0c902bfc1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -265,11 +265,18 @@ if(USE_NCCL) find_package(Nccl REQUIRED) endif() -# dmlc-core -msvc_use_static_runtime() -if(FORCE_SHARED_CRT) - set(DMLC_FORCE_SHARED_CRT ON) +if(MSVC) + if(FORCE_SHARED_CRT) + message(STATUS "XGBoost: Using dynamically linked MSVC runtime...") + set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>DLL") + else() + message(STATUS "XGBoost: Using statically linked MSVC runtime...") + set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") + endif() endif() + +# dmlc-core +set(DMLC_FORCE_SHARED_CRT ${FORCE_SHARED_CRT}) add_subdirectory(${xgboost_SOURCE_DIR}/dmlc-core) if(MSVC) @@ -490,11 +497,6 @@ if(GOOGLE_TEST) endif() endif() -# For MSVC: Call msvc_use_static_runtime() once again to completely -# replace /MD with /MT. See https://github.com/dmlc/xgboost/issues/4462 -# for issues caused by mixing of /MD and /MT flags -msvc_use_static_runtime() - # Add xgboost.pc if(ADD_PKGCONFIG) configure_file(${xgboost_SOURCE_DIR}/cmake/xgboost.pc.in ${xgboost_BINARY_DIR}/xgboost.pc @ONLY) diff --git a/cmake/Utils.cmake b/cmake/Utils.cmake index 317a71c00d22..29c53638bf2e 100644 --- a/cmake/Utils.cmake +++ b/cmake/Utils.cmake @@ -13,45 +13,6 @@ function(auto_source_group SOURCES) endforeach() endfunction() -# Force static runtime for MSVC -function(msvc_use_static_runtime) - if(MSVC AND (NOT BUILD_SHARED_LIBS) AND (NOT FORCE_SHARED_CRT)) - set(variables - CMAKE_C_FLAGS_DEBUG - CMAKE_C_FLAGS_MINSIZEREL - CMAKE_C_FLAGS_RELEASE - CMAKE_C_FLAGS_RELWITHDEBINFO - CMAKE_CXX_FLAGS_DEBUG - CMAKE_CXX_FLAGS_MINSIZEREL - CMAKE_CXX_FLAGS_RELEASE - CMAKE_CXX_FLAGS_RELWITHDEBINFO - ) - foreach(variable ${variables}) - if(${variable} MATCHES "/MD") - string(REGEX REPLACE "/MD" "/MT" ${variable} "${${variable}}") - set(${variable} "${${variable}}" PARENT_SCOPE) - endif() - endforeach() - set(variables - CMAKE_CUDA_FLAGS - CMAKE_CUDA_FLAGS_DEBUG - CMAKE_CUDA_FLAGS_MINSIZEREL - CMAKE_CUDA_FLAGS_RELEASE - CMAKE_CUDA_FLAGS_RELWITHDEBINFO - ) - foreach(variable ${variables}) - if(${variable} MATCHES "-MD") - string(REGEX REPLACE "-MD" "-MT" ${variable} "${${variable}}") - set(${variable} "${${variable}}" PARENT_SCOPE) - endif() - if(${variable} MATCHES "/MD") - string(REGEX REPLACE "/MD" "/MT" ${variable} "${${variable}}") - set(${variable} "${${variable}}" PARENT_SCOPE) - endif() - endforeach() - endif() -endfunction() - # Set output directory of target, ignoring debug or release function(set_output_directory target dir) set_target_properties(${target} PROPERTIES diff --git a/dev/prepare_jvm_release.py b/dev/prepare_jvm_release.py index 5d4d2e66fd2e..bcd119a892cb 100644 --- a/dev/prepare_jvm_release.py +++ b/dev/prepare_jvm_release.py @@ -216,8 +216,7 @@ def main(): ) print( " # Skip native build, since we have all needed native binaries from CI\n" - " export MAVEN_SKIP_NATIVE_BUILD=1\n" - " GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests" + " GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests -Dskip.native.build=true" ) print( "4. Log into https://oss.sonatype.org/. On the left menu panel, click Staging " @@ -228,9 +227,8 @@ def main(): ) print( "5. Remove the Scala 2.12 artifacts and build Scala 2.13 artifacts:\n" - " export MAVEN_SKIP_NATIVE_BUILD=1\n" " python dev/change_scala_version.py --scala-version 2.13 --purge-artifacts\n" - " GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests" + " GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests -Dskip.native.build=true" ) print( "6. Go to https://oss.sonatype.org/ to release the Scala 2.13 artifacts. " diff --git a/dmlc-core b/dmlc-core index ea21135fbb14..133418575498 160000 --- a/dmlc-core +++ b/dmlc-core @@ -1 +1 @@ -Subproject commit ea21135fbb141ae103fb5fc960289b5601b468f2 +Subproject commit 13341857549852a9a86b1894b5ba84c6276ab381 diff --git a/jvm-packages/create_jni.py b/jvm-packages/create_jni.py index 693546862b63..a4d64e1486fa 100755 --- a/jvm-packages/create_jni.py +++ b/jvm-packages/create_jni.py @@ -51,7 +51,7 @@ def maybe_makedirs(path): def run(command, **kwargs): print(command) - subprocess.check_call(command, shell=True, **kwargs) + subprocess.run(command, shell=True, check=True, env=os.environ, **kwargs) def cp(source, target): @@ -85,6 +85,8 @@ def native_build(args): if sys.platform == "linux": maybe_parallel_build = " -- -j $(nproc)" + elif sys.platform == "win32": + maybe_parallel_build = ' -- /m /nodeReuse:false "/consoleloggerparameters:ShowCommandLine;Verbosity=minimal"' else: maybe_parallel_build = "" @@ -176,15 +178,10 @@ def native_build(args): if __name__ == "__main__": - if "MAVEN_SKIP_NATIVE_BUILD" in os.environ: - print("MAVEN_SKIP_NATIVE_BUILD is set. Skipping native build...") - else: - parser = argparse.ArgumentParser() - parser.add_argument( - "--log-capi-invocation", type=str, choices=["ON", "OFF"], default="OFF" - ) - parser.add_argument( - "--use-cuda", type=str, choices=["ON", "OFF"], default="OFF" - ) - cli_args = parser.parse_args() - native_build(cli_args) + parser = argparse.ArgumentParser() + parser.add_argument( + "--log-capi-invocation", type=str, choices=["ON", "OFF"], default="OFF" + ) + parser.add_argument("--use-cuda", type=str, choices=["ON", "OFF"], default="OFF") + cli_args = parser.parse_args() + native_build(cli_args) diff --git a/jvm-packages/pom.xml b/jvm-packages/pom.xml index c8641b407b84..be9295e04b23 100644 --- a/jvm-packages/pom.xml +++ b/jvm-packages/pom.xml @@ -49,6 +49,7 @@ cuda12 3.2.18 2.12.0 + false diff --git a/jvm-packages/xgboost4j/pom.xml b/jvm-packages/xgboost4j/pom.xml index 0720d252a79c..d8a8da98c6d6 100644 --- a/jvm-packages/xgboost4j/pom.xml +++ b/jvm-packages/xgboost4j/pom.xml @@ -98,6 +98,7 @@ ${log.capi.invocation} ${user.dir} + ${skip.native.build} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 161889f9e242..49290179a899 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -30,11 +30,6 @@ if(LOG_CAPI_INVOCATION) target_compile_definitions(objxgboost PRIVATE -DLOG_CAPI_INVOCATION=1) endif() -# For MSVC: Call msvc_use_static_runtime() once again to completely -# replace /MD with /MT. See https://github.com/dmlc/xgboost/issues/4462 -# for issues caused by mixing of /MD and /MT flags -msvc_use_static_runtime() - # This grouping organises source files nicely in visual studio auto_source_group("${CUDA_SOURCES}") auto_source_group("${CPU_SOURCES}") diff --git a/src/common/io.h b/src/common/io.h index d2fcc9f92471..198ce70143be 100644 --- a/src/common/io.h +++ b/src/common/io.h @@ -48,11 +48,12 @@ struct MemoryFixSizeBuffer : public dmlc::SeekStream { curr_ptr_ += nread; return nread; } - void Write(const void *ptr, std::size_t size) override { - if (size == 0) return; + std::size_t Write(const void *ptr, std::size_t size) override { + if (size == 0) return 0; CHECK_LE(curr_ptr_ + size, buffer_size_); std::memcpy(p_buffer_ + curr_ptr_, ptr, size); curr_ptr_ += size; + return size; } void Seek(std::size_t pos) override { if (pos == kSeekEnd) { @@ -91,13 +92,14 @@ struct MemoryBufferStream : public dmlc::SeekStream { curr_ptr_ += nread; return nread; } - void Write(const void *ptr, size_t size) override { - if (size == 0) return; + std::size_t Write(const void *ptr, size_t size) override { + if (size == 0) return 0; if (curr_ptr_ + size > p_buffer_->length()) { p_buffer_->resize(curr_ptr_+size); } std::memcpy(&(*p_buffer_)[0] + curr_ptr_, ptr, size); curr_ptr_ += size; + return size; } void Seek(size_t pos) override { curr_ptr_ = static_cast(pos); @@ -127,8 +129,9 @@ class PeekableInStream : public dmlc::Stream { size_t Read(void* dptr, size_t size) override; virtual size_t PeekRead(void* dptr, size_t size); - void Write(const void*, size_t) override { + std::size_t Write(const void*, size_t) override { LOG(FATAL) << "Not implemented"; + return 0; } private: @@ -155,8 +158,9 @@ class FixedSizeStream : public PeekableInStream { [[nodiscard]] std::size_t Tell() const { return pointer_; } void Seek(size_t pos); - void Write(const void*, size_t) override { + std::size_t Write(const void*, size_t) override { LOG(FATAL) << "Not implemented"; + return 0; } /*! diff --git a/tests/buildkite/build-jvm-packages-gpu.sh b/tests/buildkite/build-jvm-packages-gpu.sh index 64be7cc0a48a..76ffafbcfdd7 100755 --- a/tests/buildkite/build-jvm-packages-gpu.sh +++ b/tests/buildkite/build-jvm-packages-gpu.sh @@ -4,7 +4,7 @@ set -euo pipefail source tests/buildkite/conftest.sh -echo "--- Build XGBoost JVM packages with CUDA" +echo "--- Build and test XGBoost JVM packages with CUDA" if [[ ($is_pull_request == 1) || ($is_release_branch == 0) ]] then diff --git a/tests/buildkite/build-jvm-packages.sh b/tests/buildkite/build-jvm-packages.sh index 1998385c548d..338a599f7e15 100755 --- a/tests/buildkite/build-jvm-packages.sh +++ b/tests/buildkite/build-jvm-packages.sh @@ -4,7 +4,7 @@ set -euo pipefail source tests/buildkite/conftest.sh -echo "--- Build XGBoost JVM packages scala 2.12" +echo "--- Build and test XGBoost JVM packages with Scala 2.12" tests/ci_build/ci_build.sh jvm tests/ci_build/build_jvm_packages.sh \ ${SPARK_VERSION} @@ -14,7 +14,7 @@ buildkite-agent artifact upload "jvm-packages/xgboost4j-spark/target/*.jar" buildkite-agent artifact upload "jvm-packages/xgboost4j-flink/target/*.jar" buildkite-agent artifact upload "jvm-packages/xgboost4j-example/target/*.jar" -echo "--- Build XGBoost JVM packages scala 2.13" +echo "--- Build and test XGBoost JVM packages with Scala 2.13" tests/ci_build/ci_build.sh jvm tests/ci_build/build_jvm_packages.sh \ ${SPARK_VERSION} "" "" "true" diff --git a/tests/buildkite/build-win64-gpu.ps1 b/tests/buildkite/build-win64-gpu.ps1 index 0799d7ac6add..9114d3237751 100644 --- a/tests/buildkite/build-win64-gpu.ps1 +++ b/tests/buildkite/build-win64-gpu.ps1 @@ -12,13 +12,11 @@ if ( $is_release_branch -eq 0 ) { } mkdir build cd build -cmake .. -G"Visual Studio 17 2022" -A x64 -DUSE_CUDA=ON -DCMAKE_VERBOSE_MAKEFILE=ON ` +cmake .. -G"Visual Studio 17 2022" -A x64 -DUSE_CUDA=ON ` -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON -DBUILD_DEPRECATED_CLI=ON ${arch_flag} -$msbuild = -join @( - "C:\\Program Files\\Microsoft Visual Studio\\2022\\Community\\MSBuild\\Current" - "\\Bin\\MSBuild.exe" -) -& $msbuild xgboost.sln /m /p:Configuration=Release /nodeReuse:false +if ($LASTEXITCODE -ne 0) { throw "Last command failed" } +cmake --build . --config Release -- /m /nodeReuse:false ` + "/consoleloggerparameters:ShowCommandLine;Verbosity=minimal" if ($LASTEXITCODE -ne 0) { throw "Last command failed" } Write-Host "--- Build binary wheel" diff --git a/tests/buildkite/infrastructure/aws-stack-creator/metadata.py b/tests/buildkite/infrastructure/aws-stack-creator/metadata.py index e086021da9f9..5012aa738854 100644 --- a/tests/buildkite/infrastructure/aws-stack-creator/metadata.py +++ b/tests/buildkite/infrastructure/aws-stack-creator/metadata.py @@ -1,27 +1,27 @@ AMI_ID = { # Managed by XGBoost team "linux-amd64-gpu": { - "us-west-2": "ami-070080d04e81c5e39", + "us-west-2": "ami-0b4079c15bbbd0faf", }, "linux-amd64-mgpu": { - "us-west-2": "ami-070080d04e81c5e39", + "us-west-2": "ami-0b4079c15bbbd0faf", }, "windows-gpu": { - "us-west-2": "ami-07c14abcf529d816a", + "us-west-2": "ami-0123456bcf4cdfb82", }, "windows-cpu": { - "us-west-2": "ami-07c14abcf529d816a", + "us-west-2": "ami-0123456bcf4cdfb82", }, # Managed by BuildKite # from https://s3.amazonaws.com/buildkite-aws-stack/latest/aws-stack.yml "linux-amd64-cpu": { - "us-west-2": "ami-0180f7fb0f07eb0bc", + "us-west-2": "ami-0083e0ae73c175ec6", }, "pipeline-loader": { - "us-west-2": "ami-0180f7fb0f07eb0bc", + "us-west-2": "ami-0083e0ae73c175ec6", }, "linux-arm64-cpu": { - "us-west-2": "ami-00686bdc2043a5505", + "us-west-2": "ami-0dbf1f9da54222f21", }, } diff --git a/tests/buildkite/infrastructure/worker-image-pipeline/windows-gpu-bootstrap.yml b/tests/buildkite/infrastructure/worker-image-pipeline/windows-gpu-bootstrap.yml index 128351e0d124..0348e28c8709 100644 --- a/tests/buildkite/infrastructure/worker-image-pipeline/windows-gpu-bootstrap.yml +++ b/tests/buildkite/infrastructure/worker-image-pipeline/windows-gpu-bootstrap.yml @@ -36,7 +36,12 @@ phases: # Install Java 11 Write-Host '>>> Installing Java 11...' - choco install openjdk11jre + choco install openjdk11 + if ($LASTEXITCODE -ne 0) { throw "Last command failed" } + + # Install Maven + Write-Host '>>> Installing Maven...' + choco install maven if ($LASTEXITCODE -ne 0) { throw "Last command failed" } # Install GraphViz diff --git a/tests/buildkite/pipeline-mgpu.yml b/tests/buildkite/pipeline-mgpu.yml index 3229646d5467..5bdf17344cba 100644 --- a/tests/buildkite/pipeline-mgpu.yml +++ b/tests/buildkite/pipeline-mgpu.yml @@ -29,7 +29,7 @@ steps: key: build-cuda agents: queue: linux-amd64-cpu - - label: ":console: Build JVM packages with CUDA" + - label: ":console: Build and test JVM packages with CUDA" command: "tests/buildkite/build-jvm-packages-gpu.sh" key: build-jvm-packages-gpu agents: