From 6d0ad77b4e3ccd00c924f354402f35fa7e412077 Mon Sep 17 00:00:00 2001 From: Nikita Titov Date: Wed, 17 Jul 2024 00:59:56 +0300 Subject: [PATCH 01/21] Replace `FetchContent_Populate` with `FetchContent_MakeAvailable` --- cmake/IntegratedOpenCL.cmake | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cmake/IntegratedOpenCL.cmake b/cmake/IntegratedOpenCL.cmake index 32ddb51d674b..02f861a92bd3 100644 --- a/cmake/IntegratedOpenCL.cmake +++ b/cmake/IntegratedOpenCL.cmake @@ -16,7 +16,7 @@ include(FetchContent) FetchContent_Declare(OpenCL-Headers GIT_REPOSITORY ${OPENCL_HEADER_REPOSITORY} GIT_TAG ${OPENCL_HEADER_TAG}) FetchContent_GetProperties(OpenCL-Headers) if(NOT OpenCL-Headers_POPULATED) - FetchContent_Populate(OpenCL-Headers) + FetchContent_MakeAvailable(OpenCL-Headers) message(STATUS "Populated OpenCL Headers") endif() set(OPENCL_ICD_LOADER_HEADERS_DIR ${opencl-headers_SOURCE_DIR} CACHE PATH "") # for OpenCL ICD Loader @@ -25,7 +25,8 @@ set(OpenCL_INCLUDE_DIR ${opencl-headers_SOURCE_DIR} CACHE PATH "") # for Boost:: FetchContent_Declare(OpenCL-ICD-Loader GIT_REPOSITORY ${OPENCL_LOADER_REPOSITORY} GIT_TAG ${OPENCL_LOADER_TAG}) FetchContent_GetProperties(OpenCL-ICD-Loader) if(NOT OpenCL-ICD-Loader_POPULATED) - FetchContent_Populate(OpenCL-ICD-Loader) + FetchContent_MakeAvailable(OpenCL-ICD-Loader) + message(STATUS "Populated OpenCL ICD Loader") if(WIN32) set(USE_DYNAMIC_VCXX_RUNTIME ON) endif() @@ -60,8 +61,10 @@ set(BOOST_INCLUDE "${BOOST_BASE}/source" CACHE PATH "") set(BOOST_LIBRARY "${BOOST_BASE}/source/stage/lib" CACHE PATH "") if(WIN32) if(MSVC) - if(${MSVC_VERSION} GREATER 1929) + if(${MSVC_VERSION} GREATER 1949) message(FATAL_ERROR "Unrecognized MSVC version number: ${MSVC_VERSION}") + elseif(${MSVC_VERSION} GREATER 1929) + set(MSVC_TOOLCHAIN_ID "v143") elseif(${MSVC_VERSION} GREATER 1919) set(MSVC_TOOLCHAIN_ID "142") elseif(${MSVC_VERSION} GREATER 1909) From 629a8164bc3cca4f7a94edefe9c457eb0b5dea1f Mon Sep 17 00:00:00 2001 From: Nikita Titov Date: Wed, 17 Jul 2024 01:03:28 +0300 Subject: [PATCH 02/21] Test with VS 2022 --- .vsts-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 169b04fffe13..23133f39b97d 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -303,7 +303,7 @@ jobs: - job: Windows ########################################### pool: - vmImage: 'windows-2019' + vmImage: 'windows-2022' strategy: matrix: regular: From 53cbe48ce5d99bb84072552859d1c83f147fca39 Mon Sep 17 00:00:00 2001 From: Nikita Titov Date: Wed, 17 Jul 2024 01:35:31 +0300 Subject: [PATCH 03/21] Update IntegratedOpenCL.cmake --- cmake/IntegratedOpenCL.cmake | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmake/IntegratedOpenCL.cmake b/cmake/IntegratedOpenCL.cmake index 02f861a92bd3..7168b8abcde7 100644 --- a/cmake/IntegratedOpenCL.cmake +++ b/cmake/IntegratedOpenCL.cmake @@ -26,11 +26,9 @@ FetchContent_Declare(OpenCL-ICD-Loader GIT_REPOSITORY ${OPENCL_LOADER_REPOSITORY FetchContent_GetProperties(OpenCL-ICD-Loader) if(NOT OpenCL-ICD-Loader_POPULATED) FetchContent_MakeAvailable(OpenCL-ICD-Loader) - message(STATUS "Populated OpenCL ICD Loader") if(WIN32) set(USE_DYNAMIC_VCXX_RUNTIME ON) endif() - add_subdirectory(${opencl-icd-loader_SOURCE_DIR} ${opencl-icd-loader_BINARY_DIR} EXCLUDE_FROM_ALL) message(STATUS "Populated OpenCL ICD Loader") endif() list(APPEND INTEGRATED_OPENCL_INCLUDES ${OPENCL_ICD_LOADER_HEADERS_DIR}) From e923d7750e6dc23ab73ceb6c0e34d33e99af5847 Mon Sep 17 00:00:00 2001 From: Nikita Titov Date: Wed, 17 Jul 2024 02:16:06 +0300 Subject: [PATCH 04/21] Update IntegratedOpenCL.cmake --- cmake/IntegratedOpenCL.cmake | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/cmake/IntegratedOpenCL.cmake b/cmake/IntegratedOpenCL.cmake index 7168b8abcde7..4fa9302f9c5a 100644 --- a/cmake/IntegratedOpenCL.cmake +++ b/cmake/IntegratedOpenCL.cmake @@ -22,10 +22,19 @@ endif() set(OPENCL_ICD_LOADER_HEADERS_DIR ${opencl-headers_SOURCE_DIR} CACHE PATH "") # for OpenCL ICD Loader set(OpenCL_INCLUDE_DIR ${opencl-headers_SOURCE_DIR} CACHE PATH "") # for Boost::Compute -FetchContent_Declare(OpenCL-ICD-Loader GIT_REPOSITORY ${OPENCL_LOADER_REPOSITORY} GIT_TAG ${OPENCL_LOADER_TAG}) +if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.28) + FetchContent_Declare(OpenCL-ICD-Loader GIT_REPOSITORY ${OPENCL_LOADER_REPOSITORY} GIT_TAG ${OPENCL_LOADER_TAG} EXCLUDE_FROM_ALL) +else() + FetchContent_Declare(OpenCL-ICD-Loader GIT_REPOSITORY ${OPENCL_LOADER_REPOSITORY} GIT_TAG ${OPENCL_LOADER_TAG}) +endif() FetchContent_GetProperties(OpenCL-ICD-Loader) if(NOT OpenCL-ICD-Loader_POPULATED) - FetchContent_MakeAvailable(OpenCL-ICD-Loader) + if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.28) + FetchContent_MakeAvailable(OpenCL-ICD-Loader) + else() + FetchContent_Populate(OpenCL-ICD-Loader) + add_subdirectory(${opencl-icd-loader_SOURCE_DIR} ${opencl-icd-loader_BINARY_DIR} EXCLUDE_FROM_ALL) + endif() if(WIN32) set(USE_DYNAMIC_VCXX_RUNTIME ON) endif() From c97def02f30f17c3500057bb6ffc8dc440dd730e Mon Sep 17 00:00:00 2001 From: Nikita Titov Date: Wed, 17 Jul 2024 02:39:07 +0300 Subject: [PATCH 05/21] Update IntegratedOpenCL.cmake --- cmake/IntegratedOpenCL.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/IntegratedOpenCL.cmake b/cmake/IntegratedOpenCL.cmake index 4fa9302f9c5a..a40b0ad56141 100644 --- a/cmake/IntegratedOpenCL.cmake +++ b/cmake/IntegratedOpenCL.cmake @@ -1,5 +1,5 @@ set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) -set(BOOST_VERSION_DOT "1.74") +set(BOOST_VERSION_DOT "1.85") string(REPLACE "." "_" BOOST_VERSION_UNDERSCORE ${BOOST_VERSION_DOT}) set(OPENCL_HEADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-Headers.git") From 70df6df1bcc96d8541dcf120757e79d3bcf257cc Mon Sep 17 00:00:00 2001 From: Nikita Titov Date: Wed, 17 Jul 2024 02:55:41 +0300 Subject: [PATCH 06/21] Update IntegratedOpenCL.cmake --- cmake/IntegratedOpenCL.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmake/IntegratedOpenCL.cmake b/cmake/IntegratedOpenCL.cmake index a40b0ad56141..20803e679f29 100644 --- a/cmake/IntegratedOpenCL.cmake +++ b/cmake/IntegratedOpenCL.cmake @@ -114,6 +114,7 @@ list( "libs/any" "libs/array" "libs/assert" + "libs/atomic" "libs/bind" "libs/chrono" "libs/compute" @@ -181,6 +182,7 @@ ExternalProject_Add( -a -q -j ${J} + --with-atomic --with-headers --with-chrono --with-filesystem From 72324f2156e3947bac2657e76674d0825a45f2cd Mon Sep 17 00:00:00 2001 From: Nikita Titov Date: Wed, 17 Jul 2024 03:03:19 +0300 Subject: [PATCH 07/21] Update IntegratedOpenCL.cmake --- cmake/IntegratedOpenCL.cmake | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/cmake/IntegratedOpenCL.cmake b/cmake/IntegratedOpenCL.cmake index 20803e679f29..446ddb6b4d14 100644 --- a/cmake/IntegratedOpenCL.cmake +++ b/cmake/IntegratedOpenCL.cmake @@ -1,5 +1,5 @@ set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) -set(BOOST_VERSION_DOT "1.85") +set(BOOST_VERSION_DOT "1.74") string(REPLACE "." "_" BOOST_VERSION_UNDERSCORE ${BOOST_VERSION_DOT}) set(OPENCL_HEADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-Headers.git") @@ -68,10 +68,8 @@ set(BOOST_INCLUDE "${BOOST_BASE}/source" CACHE PATH "") set(BOOST_LIBRARY "${BOOST_BASE}/source/stage/lib" CACHE PATH "") if(WIN32) if(MSVC) - if(${MSVC_VERSION} GREATER 1949) + if(${MSVC_VERSION} GREATER 1929) message(FATAL_ERROR "Unrecognized MSVC version number: ${MSVC_VERSION}") - elseif(${MSVC_VERSION} GREATER 1929) - set(MSVC_TOOLCHAIN_ID "v143") elseif(${MSVC_VERSION} GREATER 1919) set(MSVC_TOOLCHAIN_ID "142") elseif(${MSVC_VERSION} GREATER 1909) @@ -114,7 +112,6 @@ list( "libs/any" "libs/array" "libs/assert" - "libs/atomic" "libs/bind" "libs/chrono" "libs/compute" @@ -182,7 +179,6 @@ ExternalProject_Add( -a -q -j ${J} - --with-atomic --with-headers --with-chrono --with-filesystem From a8616c598546311fe220bfc12470ea6235ec525f Mon Sep 17 00:00:00 2001 From: Nikita Titov Date: Wed, 17 Jul 2024 03:04:10 +0300 Subject: [PATCH 08/21] Update .vsts-ci.yml --- .vsts-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 23133f39b97d..169b04fffe13 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -303,7 +303,7 @@ jobs: - job: Windows ########################################### pool: - vmImage: 'windows-2022' + vmImage: 'windows-2019' strategy: matrix: regular: From 639db609bf0dea35a3c7bf76c249f2a5cec6ed86 Mon Sep 17 00:00:00 2001 From: StrikerRUS Date: Wed, 17 Jul 2024 14:59:18 +0300 Subject: [PATCH 09/21] bump minimum CMake version --- .ci/setup.sh | 19 ++++++++----------- .ci/test_r_package.sh | 10 ---------- .github/workflows/r_package.yml | 5 ----- CMakeLists.txt | 2 +- cmake/IntegratedOpenCL.cmake | 13 ++----------- python-package/pyproject.toml | 2 +- 6 files changed, 12 insertions(+), 39 deletions(-) diff --git a/.ci/setup.sh b/.ci/setup.sh index fe918d68a78c..32d2e185c3d7 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -29,6 +29,13 @@ if [[ $OS_NAME == "macos" ]]; then brew install swig fi else # Linux + curl -O -L \ + https://github.com/Kitware/CMake/releases/download/v3.30.0/cmake-3.30.0-linux-${ARCH}.sh \ + || exit 1 + sudo mkdir /opt/cmake || exit 1 + sudo sh cmake-3.30.0-linux-${ARCH}.sh --skip-license --prefix=/opt/cmake || exit 1 + sudo ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake || exit 1 + if [[ $IN_UBUNTU_BASE_CONTAINER == "true" ]]; then # fixes error "unable to initialize frontend: Dialog" # https://github.com/moby/moby/issues/27988#issuecomment-462809153 @@ -41,7 +48,6 @@ else # Linux sudo apt-get install --no-install-recommends -y \ build-essential \ ca-certificates \ - cmake \ curl \ git \ libcurl4 \ @@ -117,21 +123,12 @@ else # Linux fi if [[ $TASK == "cuda" ]]; then echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections - apt-get update - apt-get install --no-install-recommends -y \ - curl \ - lsb-release \ - software-properties-common if [[ $COMPILER == "clang" ]]; then + apt-get update apt-get install --no-install-recommends -y \ clang \ libomp-dev fi - curl -sL https://apt.kitware.com/keys/kitware-archive-latest.asc | apt-key add - - apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" -y - apt-get update - apt-get install --no-install-recommends -y \ - cmake fi fi diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index a68e006e8036..5eba7669ad14 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -4,7 +4,6 @@ set -e -E -u -o pipefail # defaults ARCH=$(uname -m) -INSTALL_CMAKE_FROM_RELEASES=${INSTALL_CMAKE_FROM_RELEASES:-"false"} # set up R environment CRAN_MIRROR="https://cran.rstudio.com" @@ -74,15 +73,6 @@ if [[ $OS_NAME == "linux" ]]; then automake \ || exit 1 fi - if [[ $INSTALL_CMAKE_FROM_RELEASES == "true" ]]; then - curl -O -L \ - https://github.com/Kitware/CMake/releases/download/v3.25.1/cmake-3.25.1-linux-${ARCH}.sh \ - || exit 1 - - sudo mkdir /opt/cmake || exit 1 - sudo sh cmake-3.25.1-linux-${ARCH}.sh --skip-license --prefix=/opt/cmake || exit 1 - sudo ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake || exit 1 - fi fi # Installing R precompiled for Mac OS 10.11 or higher diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml index fd456776766b..e7e6eb5e499b 100644 --- a/.github/workflows/r_package.yml +++ b/.github/workflows/r_package.yml @@ -204,11 +204,6 @@ jobs: elif [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then export OS_NAME="linux" export IN_UBUNTU_BASE_CONTAINER="true" - # the default version of cmake provided on Ubuntu 18.04 (v3.10.2), is not supported by LightGBM - # see https://github.com/microsoft/LightGBM/issues/5642 - if [[ "${{ matrix.container }}" == "ubuntu:18.04" ]]; then - export INSTALL_CMAKE_FROM_RELEASES="true" - fi fi export BUILD_DIRECTORY="$GITHUB_WORKSPACE" export R_VERSION="${{ matrix.r_version }}" diff --git a/CMakeLists.txt b/CMakeLists.txt index 982535b7258c..fb8daa9a104e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,7 +23,7 @@ option(__BUILD_FOR_PYTHON "Set to ON if building lib_lightgbm for use with the P option(__BUILD_FOR_R "Set to ON if building lib_lightgbm for use with the R package" OFF) option(__INTEGRATE_OPENCL "Set to ON if building LightGBM with the OpenCL ICD Loader and its dependencies included" OFF) -cmake_minimum_required(VERSION 3.18) +cmake_minimum_required(VERSION 3.28) # If using Visual Studio generators, always target v10.x of the Windows SDK. # Doing this avoids lookups that could fall back to very old versions, e.g. by finding diff --git a/cmake/IntegratedOpenCL.cmake b/cmake/IntegratedOpenCL.cmake index 446ddb6b4d14..6d9b0f0e8980 100644 --- a/cmake/IntegratedOpenCL.cmake +++ b/cmake/IntegratedOpenCL.cmake @@ -22,19 +22,10 @@ endif() set(OPENCL_ICD_LOADER_HEADERS_DIR ${opencl-headers_SOURCE_DIR} CACHE PATH "") # for OpenCL ICD Loader set(OpenCL_INCLUDE_DIR ${opencl-headers_SOURCE_DIR} CACHE PATH "") # for Boost::Compute -if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.28) - FetchContent_Declare(OpenCL-ICD-Loader GIT_REPOSITORY ${OPENCL_LOADER_REPOSITORY} GIT_TAG ${OPENCL_LOADER_TAG} EXCLUDE_FROM_ALL) -else() - FetchContent_Declare(OpenCL-ICD-Loader GIT_REPOSITORY ${OPENCL_LOADER_REPOSITORY} GIT_TAG ${OPENCL_LOADER_TAG}) -endif() +FetchContent_Declare(OpenCL-ICD-Loader GIT_REPOSITORY ${OPENCL_LOADER_REPOSITORY} GIT_TAG ${OPENCL_LOADER_TAG} EXCLUDE_FROM_ALL) FetchContent_GetProperties(OpenCL-ICD-Loader) if(NOT OpenCL-ICD-Loader_POPULATED) - if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.28) - FetchContent_MakeAvailable(OpenCL-ICD-Loader) - else() - FetchContent_Populate(OpenCL-ICD-Loader) - add_subdirectory(${opencl-icd-loader_SOURCE_DIR} ${opencl-icd-loader_BINARY_DIR} EXCLUDE_FROM_ALL) - endif() + FetchContent_MakeAvailable(OpenCL-ICD-Loader) if(WIN32) set(USE_DYNAMIC_VCXX_RUNTIME ON) endif() diff --git a/python-package/pyproject.toml b/python-package/pyproject.toml index 39fc6d8f0566..21462911999f 100644 --- a/python-package/pyproject.toml +++ b/python-package/pyproject.toml @@ -63,7 +63,7 @@ build-backend = "scikit_build_core.build" # based on https://github.com/scikit-build/scikit-build-core#configuration [tool.scikit-build] -cmake.version = ">=3.18" +cmake.version = ">=3.28" ninja.version = ">=1.11" ninja.make-fallback = true cmake.args = [ From 03af08de2bf88bfe7f19ba1dec6b2d8c417376e6 Mon Sep 17 00:00:00 2001 From: StrikerRUS Date: Wed, 17 Jul 2024 15:12:14 +0300 Subject: [PATCH 10/21] force symlinking --- .ci/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/setup.sh b/.ci/setup.sh index 32d2e185c3d7..d785ffe8f05d 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -34,7 +34,7 @@ else # Linux || exit 1 sudo mkdir /opt/cmake || exit 1 sudo sh cmake-3.30.0-linux-${ARCH}.sh --skip-license --prefix=/opt/cmake || exit 1 - sudo ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake || exit 1 + sudo ln -sf /opt/cmake/bin/cmake /usr/local/bin/cmake || exit 1 if [[ $IN_UBUNTU_BASE_CONTAINER == "true" ]]; then # fixes error "unable to initialize frontend: Dialog" From b8c2b9d0d98d05faee5ff2de3fe6ca0f3069f0d7 Mon Sep 17 00:00:00 2001 From: StrikerRUS Date: Wed, 17 Jul 2024 15:31:14 +0300 Subject: [PATCH 11/21] fix line length --- cmake/IntegratedOpenCL.cmake | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cmake/IntegratedOpenCL.cmake b/cmake/IntegratedOpenCL.cmake index 6d9b0f0e8980..2397a7e7c26d 100644 --- a/cmake/IntegratedOpenCL.cmake +++ b/cmake/IntegratedOpenCL.cmake @@ -22,7 +22,14 @@ endif() set(OPENCL_ICD_LOADER_HEADERS_DIR ${opencl-headers_SOURCE_DIR} CACHE PATH "") # for OpenCL ICD Loader set(OpenCL_INCLUDE_DIR ${opencl-headers_SOURCE_DIR} CACHE PATH "") # for Boost::Compute -FetchContent_Declare(OpenCL-ICD-Loader GIT_REPOSITORY ${OPENCL_LOADER_REPOSITORY} GIT_TAG ${OPENCL_LOADER_TAG} EXCLUDE_FROM_ALL) +FetchContent_Declare( + OpenCL-ICD-Loader + GIT_REPOSITORY + ${OPENCL_LOADER_REPOSITORY} + GIT_TAG + ${OPENCL_LOADER_TAG} + EXCLUDE_FROM_ALL +) FetchContent_GetProperties(OpenCL-ICD-Loader) if(NOT OpenCL-ICD-Loader_POPULATED) FetchContent_MakeAvailable(OpenCL-ICD-Loader) From 151b8e917aed2874cda3f32fd77e7cace07aa2b5 Mon Sep 17 00:00:00 2001 From: StrikerRUS Date: Wed, 17 Jul 2024 16:07:40 +0300 Subject: [PATCH 12/21] install curl --- .ci/setup.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.ci/setup.sh b/.ci/setup.sh index d785ffe8f05d..1c3c15e48083 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -29,6 +29,15 @@ if [[ $OS_NAME == "macos" ]]; then brew install swig fi else # Linux + if type -f apt 2>&1 > /dev/null; then + apt-get update + apt-get install --no-install-recommends -y \ + curl + else + yum update -y + yum install -y \ + curl + fi curl -O -L \ https://github.com/Kitware/CMake/releases/download/v3.30.0/cmake-3.30.0-linux-${ARCH}.sh \ || exit 1 From f383a935723fe1e1bcc236d272bec0cace969e8f Mon Sep 17 00:00:00 2001 From: StrikerRUS Date: Wed, 17 Jul 2024 16:58:47 +0300 Subject: [PATCH 13/21] install sudo --- .ci/setup.sh | 8 ++++---- .github/workflows/cuda.yml | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.ci/setup.sh b/.ci/setup.sh index 1c3c15e48083..a0db1af65fa6 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -30,12 +30,12 @@ if [[ $OS_NAME == "macos" ]]; then fi else # Linux if type -f apt 2>&1 > /dev/null; then - apt-get update - apt-get install --no-install-recommends -y \ + sudo apt-get update + sudo apt-get install --no-install-recommends -y \ curl else - yum update -y - yum install -y \ + sudo yum update -y + sudo yum install -y \ curl fi curl -O -L \ diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 735bcef4a0d7..6c32db1a2ed8 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -98,7 +98,7 @@ jobs: linux_version: "ubuntu20.04" task: cuda steps: - - name: Install latest git + - name: Install latest git and sudo run: | apt-get update apt-get install --no-install-recommends -y \ @@ -107,7 +107,8 @@ jobs: add-apt-repository ppa:git-core/ppa -y apt-get update apt-get install --no-install-recommends -y \ - git + git \ + sudo - name: Checkout repository uses: actions/checkout@v4 with: From 64a528a1e0485048f4f15b5e275e05bf03c6c9e8 Mon Sep 17 00:00:00 2001 From: StrikerRUS Date: Wed, 17 Jul 2024 17:39:04 +0300 Subject: [PATCH 14/21] install certs --- .ci/setup.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.ci/setup.sh b/.ci/setup.sh index a0db1af65fa6..1a49d968695a 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -32,10 +32,12 @@ else # Linux if type -f apt 2>&1 > /dev/null; then sudo apt-get update sudo apt-get install --no-install-recommends -y \ + ca-certificates \ curl else sudo yum update -y sudo yum install -y \ + ca-certificates \ curl fi curl -O -L \ From 571a7beaadb96cdde36bd2dceb337cab58aefa13 Mon Sep 17 00:00:00 2001 From: StrikerRUS Date: Wed, 17 Jul 2024 19:21:37 +0300 Subject: [PATCH 15/21] install CMake for R Windows jobs --- .ci/test_r_package_windows.ps1 | 17 ++++++++++++++--- .github/workflows/r_package.yml | 2 +- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index 6af4dbb04173..6f0db7fc3918 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -45,6 +45,8 @@ Remove-From-Path ".*android.*" Remove-From-Path ".*Android.*" Remove-From-Path ".*chocolatey.*" Remove-From-Path ".*Chocolatey.*" +Remove-From-Path ".*cmake.*" +Remove-From-Path ".*CMake.*" Remove-From-Path ".*\\Git\\.*" Remove-From-Path "(?!.*pandoc.*).*hostedtoolcache.*" Remove-From-Path ".*Microsoft SDKs.*" @@ -87,10 +89,12 @@ if ($env:R_MAJOR_VERSION -eq "3") { Write-Output "[ERROR] Unrecognized R version: $env:R_VERSION" Check-Output $false } +$env:CMAKE_VERSION = "3.30.0" $env:R_LIB_PATH = "$env:BUILD_SOURCESDIRECTORY/RLibrary" -replace '[\\]', '/' $env:R_LIBS = "$env:R_LIB_PATH" -$env:PATH = "$env:RTOOLS_BIN;" + "$env:RTOOLS_MINGW_BIN;" + "$env:R_LIB_PATH/R/bin/x64;"+ $env:PATH +$env:CMAKE_PATH = "$env:BUILD_SOURCESDIRECTORY/CMake_installation" +$env:PATH = "$env:RTOOLS_BIN;" + "$env:RTOOLS_MINGW_BIN;" + "$env:R_LIB_PATH/R/bin/x64;" + "$env:CMAKE_PATH/cmake-$env:CMAKE_VERSION-windows-x86_64/bin;" + $env:PATH if ([version]$env:R_VERSION -lt [version]"4.0") { $env:CRAN_MIRROR = "https://cran-archive.r-project.org" } else { @@ -112,11 +116,13 @@ if (($env:COMPILER -eq "MINGW") -and ($env:R_BUILD_TYPE -eq "cmake")) { cd $env:BUILD_SOURCESDIRECTORY tzutil /s "GMT Standard Time" [Void][System.IO.Directory]::CreateDirectory($env:R_LIB_PATH) +[Void][System.IO.Directory]::CreateDirectory($env:CMAKE_PATH) -# download R and RTools -Write-Output "Downloading R and Rtools" +# download R, RTools and CMake +Write-Output "Downloading R, Rtools and CMake" Download-File-With-Retries -url "$env:CRAN_MIRROR/bin/windows/base/old/$env:R_WINDOWS_VERSION/R-$env:R_WINDOWS_VERSION-win.exe" -destfile "R-win.exe" Download-File-With-Retries -url "https://github.com/microsoft/LightGBM/releases/download/v2.0.12/$env:RTOOLS_EXE_FILE" -destfile "Rtools.exe" +Download-File-With-Retries -url "https://github.com/Kitware/CMake/releases/download/v$env:CMAKE_VERSION/cmake-$env:CMAKE_VERSION-windows-x86_64.zip" -destfile "$env:CMAKE_PATH/cmake.zip" # Install R Write-Output "Installing R" @@ -127,6 +133,11 @@ Write-Output "Installing Rtools" Start-Process -FilePath Rtools.exe -NoNewWindow -Wait -ArgumentList "/VERYSILENT /SUPPRESSMSGBOXES /DIR=$RTOOLS_INSTALL_PATH" ; Check-Output $? Write-Output "Done installing Rtools" +Write-Output "Installing CMake" +Add-Type -AssemblyName System.IO.Compression.FileSystem +[System.IO.Compression.ZipFile]::ExtractToDirectory("$env:CMake_PATH/cmake.zip", "$env:CMake_PATH") +Write-Output "Done installing CMake" + Write-Output "Installing dependencies" $packages = "c('data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'processx', 'R6', 'RhpcBLASctl', 'testthat'), dependencies = c('Imports', 'Depends', 'LinkingTo')" Run-R-Code-Redirect-Stderr "options(install.packages.check.source = 'no'); install.packages($packages, repos = '$env:CRAN_MIRROR', type = 'binary', lib = '$env:R_LIB_PATH', Ncpus = parallel::detectCores())" ; Check-Output $? diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml index e7e6eb5e499b..9aaf4cd72872 100644 --- a/.github/workflows/r_package.yml +++ b/.github/workflows/r_package.yml @@ -181,7 +181,7 @@ jobs: - name: Install pandoc uses: r-lib/actions/setup-pandoc@v2 if: matrix.container != 'ubuntu:18.04' - # R 3.6 binary isn't easily available on buntu 18.04, + # R 3.6 binary isn't easily available on Ubuntu 18.04, # but setup-pandoc>=2.7.1 is uses a too-new glibc for it. # ref: https://github.com/microsoft/LightGBM/issues/6298 - name: Install pandoc From 1fe7190c5e1062e710d13cce2e2e7b4d10ad4651 Mon Sep 17 00:00:00 2001 From: StrikerRUS Date: Wed, 17 Jul 2024 19:35:39 +0300 Subject: [PATCH 16/21] debug cmake --- .ci/test_r_package_windows.ps1 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index 6f0db7fc3918..74c304b80b5a 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -35,6 +35,8 @@ function Remove-From-Path { ) $env:PATH = ($env:PATH.Split(';') | Where-Object { $_ -notmatch "$pattern_to_remove" }) -join ';' } +Write-Output "Finding CMake" +where cmake # remove some details that exist in the GitHub Actions images which might # cause conflicts with R and other components installed by this script @@ -135,7 +137,7 @@ Write-Output "Done installing Rtools" Write-Output "Installing CMake" Add-Type -AssemblyName System.IO.Compression.FileSystem -[System.IO.Compression.ZipFile]::ExtractToDirectory("$env:CMake_PATH/cmake.zip", "$env:CMake_PATH") +[System.IO.Compression.ZipFile]::ExtractToDirectory("$env:CMake_PATH/cmake.zip", "$env:CMake_PATH") ; Check-Output $? Write-Output "Done installing CMake" Write-Output "Installing dependencies" From be486022cd78acc2d10f4f4a9fe3c62713b62f5d Mon Sep 17 00:00:00 2001 From: StrikerRUS Date: Thu, 18 Jul 2024 00:15:57 +0300 Subject: [PATCH 17/21] new try --- .ci/test_r_package_windows.ps1 | 10 ++++++---- .github/workflows/r_package.yml | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index 74c304b80b5a..45e076f9478b 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -35,8 +35,6 @@ function Remove-From-Path { ) $env:PATH = ($env:PATH.Split(';') | Where-Object { $_ -notmatch "$pattern_to_remove" }) -join ';' } -Write-Output "Finding CMake" -where cmake # remove some details that exist in the GitHub Actions images which might # cause conflicts with R and other components installed by this script @@ -95,7 +93,7 @@ $env:CMAKE_VERSION = "3.30.0" $env:R_LIB_PATH = "$env:BUILD_SOURCESDIRECTORY/RLibrary" -replace '[\\]', '/' $env:R_LIBS = "$env:R_LIB_PATH" -$env:CMAKE_PATH = "$env:BUILD_SOURCESDIRECTORY/CMake_installation" +$env:CMAKE_PATH = "$env:BUILD_SOURCESDIRECTORY/CMake_installation" -replace '[\\]', '/' $env:PATH = "$env:RTOOLS_BIN;" + "$env:RTOOLS_MINGW_BIN;" + "$env:R_LIB_PATH/R/bin/x64;" + "$env:CMAKE_PATH/cmake-$env:CMAKE_VERSION-windows-x86_64/bin;" + $env:PATH if ([version]$env:R_VERSION -lt [version]"4.0") { $env:CRAN_MIRROR = "https://cran-archive.r-project.org" @@ -137,9 +135,13 @@ Write-Output "Done installing Rtools" Write-Output "Installing CMake" Add-Type -AssemblyName System.IO.Compression.FileSystem -[System.IO.Compression.ZipFile]::ExtractToDirectory("$env:CMake_PATH/cmake.zip", "$env:CMake_PATH") ; Check-Output $? +[System.IO.Compression.ZipFile]::ExtractToDirectory("$env:CMAKE_PATH/cmake.zip", "$env:CMAKE_PATH") ; Check-Output $? Write-Output "Done installing CMake" +Write-Output "Finding CMake" +cmake --version +(Get-Command cmake).Path + Write-Output "Installing dependencies" $packages = "c('data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'processx', 'R6', 'RhpcBLASctl', 'testthat'), dependencies = c('Imports', 'Depends', 'LinkingTo')" Run-R-Code-Redirect-Stderr "options(install.packages.check.source = 'no'); install.packages($packages, repos = '$env:CRAN_MIRROR', type = 'binary', lib = '$env:R_LIB_PATH', Ncpus = parallel::detectCores())" ; Check-Output $? diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml index 9aaf4cd72872..7ebd769ace1f 100644 --- a/.github/workflows/r_package.yml +++ b/.github/workflows/r_package.yml @@ -187,7 +187,7 @@ jobs: - name: Install pandoc uses: r-lib/actions/setup-pandoc@v2.6.0 if: matrix.container == 'ubuntu:18.04' - - name: install tinytex + - name: Install tinytex if: startsWith(matrix.os, 'windows') uses: r-lib/actions/setup-tinytex@v2 env: From 93d561fdefb74fbdfe6ea5b8f4bbaee7b01d3453 Mon Sep 17 00:00:00 2001 From: StrikerRUS Date: Thu, 18 Jul 2024 00:38:19 +0300 Subject: [PATCH 18/21] remove CMake from RTools --- .ci/test_r_package_windows.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index 45e076f9478b..aed490521111 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -136,6 +136,8 @@ Write-Output "Done installing Rtools" Write-Output "Installing CMake" Add-Type -AssemblyName System.IO.Compression.FileSystem [System.IO.Compression.ZipFile]::ExtractToDirectory("$env:CMAKE_PATH/cmake.zip", "$env:CMAKE_PATH") ; Check-Output $? +# Remove old CMake shiped with RTools +Remove-Item "$env:RTOOLS_MINGW_BIN/cmake.exe" -Force -ErrorAction Ignore Write-Output "Done installing CMake" Write-Output "Finding CMake" From 97c8561de7f3b90d014a94646b9725c6f3a8ae77 Mon Sep 17 00:00:00 2001 From: StrikerRUS Date: Thu, 18 Jul 2024 01:08:05 +0300 Subject: [PATCH 19/21] cleanup --- .ci/test_r_package_windows.ps1 | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index aed490521111..f0b71606934c 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -93,7 +93,7 @@ $env:CMAKE_VERSION = "3.30.0" $env:R_LIB_PATH = "$env:BUILD_SOURCESDIRECTORY/RLibrary" -replace '[\\]', '/' $env:R_LIBS = "$env:R_LIB_PATH" -$env:CMAKE_PATH = "$env:BUILD_SOURCESDIRECTORY/CMake_installation" -replace '[\\]', '/' +$env:CMAKE_PATH = "$env:BUILD_SOURCESDIRECTORY/CMake_installation" $env:PATH = "$env:RTOOLS_BIN;" + "$env:RTOOLS_MINGW_BIN;" + "$env:R_LIB_PATH/R/bin/x64;" + "$env:CMAKE_PATH/cmake-$env:CMAKE_VERSION-windows-x86_64/bin;" + $env:PATH if ([version]$env:R_VERSION -lt [version]"4.0") { $env:CRAN_MIRROR = "https://cran-archive.r-project.org" @@ -140,10 +140,6 @@ Add-Type -AssemblyName System.IO.Compression.FileSystem Remove-Item "$env:RTOOLS_MINGW_BIN/cmake.exe" -Force -ErrorAction Ignore Write-Output "Done installing CMake" -Write-Output "Finding CMake" -cmake --version -(Get-Command cmake).Path - Write-Output "Installing dependencies" $packages = "c('data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'processx', 'R6', 'RhpcBLASctl', 'testthat'), dependencies = c('Imports', 'Depends', 'LinkingTo')" Run-R-Code-Redirect-Stderr "options(install.packages.check.source = 'no'); install.packages($packages, repos = '$env:CRAN_MIRROR', type = 'binary', lib = '$env:R_LIB_PATH', Ncpus = parallel::detectCores())" ; Check-Output $? From fa116f0a18c609aaa23a6f8689efd57b9e40538e Mon Sep 17 00:00:00 2001 From: Nikita Titov Date: Wed, 24 Jul 2024 19:42:32 +0300 Subject: [PATCH 20/21] remove duplicted curl installation --- .ci/setup.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.ci/setup.sh b/.ci/setup.sh index 1a49d968695a..8bb3b07c91e1 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -58,8 +58,6 @@ else # Linux sudo apt-get install --no-install-recommends -y \ build-essential \ - ca-certificates \ - curl \ git \ libcurl4 \ libicu-dev \ From 98b853e28252ceb6a4074412165f4c4c95e5bb59 Mon Sep 17 00:00:00 2001 From: Nikita Titov Date: Fri, 2 Aug 2024 13:50:39 +0300 Subject: [PATCH 21/21] CMake version as variable --- .ci/setup.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.ci/setup.sh b/.ci/setup.sh index 8bb3b07c91e1..53976e216815 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -40,11 +40,12 @@ else # Linux ca-certificates \ curl fi + CMAKE_VERSION="3.30.0" curl -O -L \ - https://github.com/Kitware/CMake/releases/download/v3.30.0/cmake-3.30.0-linux-${ARCH}.sh \ + https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${ARCH}.sh \ || exit 1 sudo mkdir /opt/cmake || exit 1 - sudo sh cmake-3.30.0-linux-${ARCH}.sh --skip-license --prefix=/opt/cmake || exit 1 + sudo sh cmake-${CMAKE_VERSION}-linux-${ARCH}.sh --skip-license --prefix=/opt/cmake || exit 1 sudo ln -sf /opt/cmake/bin/cmake /usr/local/bin/cmake || exit 1 if [[ $IN_UBUNTU_BASE_CONTAINER == "true" ]]; then