Skip to content

Commit

Permalink
Merge branch 'master' into dask
Browse files Browse the repository at this point in the history
  • Loading branch information
jameslamb committed May 10, 2021
2 parents f058167 + 08d1ce4 commit 23521a5
Show file tree
Hide file tree
Showing 126 changed files with 2,266 additions and 1,576 deletions.
2 changes: 1 addition & 1 deletion .appveyor.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
version: 3.2.0.99.{build}
version: 3.2.1.99.{build}

image: Visual Studio 2015
platform: x64
Expand Down
35 changes: 21 additions & 14 deletions .ci/setup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ if [[ $OS_NAME == "macos" ]]; then
if [[ $TASK == "swig" ]]; then
brew install swig
fi
brew install graphviz
curl -sL -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
else # Linux
if [[ $IN_UBUNTU_LATEST_CONTAINER == "true" ]]; then
Expand All @@ -25,13 +26,10 @@ else # Linux
echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections

sudo apt-get update
sudo apt-get install -y --no-install-recommends \
sudo apt-get install --no-install-recommends -y \
software-properties-common

sudo add-apt-repository -y ppa:git-core/ppa
sudo apt-get update

sudo apt-get install -y --no-install-recommends \
sudo apt-get install --no-install-recommends -y \
apt-utils \
build-essential \
ca-certificates \
Expand Down Expand Up @@ -64,12 +62,16 @@ else # Linux
fi
if [[ $TASK == "mpi" ]]; then
sudo apt-get update
sudo apt-get install --no-install-recommends -y libopenmpi-dev openmpi-bin
sudo apt-get install --no-install-recommends -y \
libopenmpi-dev \
openmpi-bin
fi
if [[ $TASK == "gpu" ]]; then
sudo add-apt-repository ppa:mhier/libboost-latest -y
sudo apt-get update
sudo apt-get install --no-install-recommends -y libboost1.74-dev ocl-icd-opencl-dev
sudo apt-get install --no-install-recommends -y \
libboost1.74-dev \
ocl-icd-opencl-dev
cd $BUILD_DIRECTORY # to avoid permission errors
curl -sL -o AMD-APP-SDKInstaller.tar.bz2 https://github.com/microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.136-GA-linux64.tar.bz2
tar -xjf AMD-APP-SDKInstaller.tar.bz2
Expand All @@ -79,11 +81,13 @@ else # Linux
mv $AMDAPPSDK_PATH/lib/x86_64/sdk/* $AMDAPPSDK_PATH/lib/x86_64/
echo libamdocl64.so > $OPENCL_VENDOR_PATH/amdocl64.icd
fi
ARCH=$(uname -m)
if [[ $TASK == "cuda" ]]; then
echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
apt-get update
apt-get install --no-install-recommends -y \
curl \
graphviz \
libxau6 \
libxext6 \
libxrender1 \
Expand All @@ -97,17 +101,20 @@ else # Linux
curl -sL https://apt.kitware.com/keys/kitware-archive-latest.asc | apt-key add -
apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" -y
apt-get update
if [[ $COMPILER == "clang" ]]; then
apt-get install --no-install-recommends -y \
cmake="3.19.5-0kitware1" \
cmake-data="3.19.5-0kitware1"
apt-get install --no-install-recommends -y \
cmake
else
if [[ $ARCH != "x86_64" ]]; then
yum update -y
yum install -y \
graphviz
else
apt-get install --no-install-recommends -y \
cmake
sudo apt-get update
sudo apt-get install --no-install-recommends -y \
graphviz
fi
fi
if [[ $SETUP_CONDA != "false" ]]; then
ARCH=$(uname -m)
if [[ $ARCH == "x86_64" ]]; then
curl -sL -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
else
Expand Down
20 changes: 11 additions & 9 deletions .ci/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,14 @@ if [[ "${TASK}" == "r-package" ]]; then
exit 0
fi

if [[ "$TASK" == "cpp-tests" ]]; then
mkdir $BUILD_DIRECTORY/build && cd $BUILD_DIRECTORY/build
cmake -DBUILD_CPP_TEST=ON -DUSE_OPENMP=OFF ..
make testlightgbm -j4 || exit -1
./../testlightgbm || exit -1
exit 0
fi

conda create -q -y -n $CONDA_ENV python=$PYTHON_VERSION
source activate $CONDA_ENV

Expand Down Expand Up @@ -71,9 +79,9 @@ fi
if [[ $TASK == "if-else" ]]; then
conda install -q -y -n $CONDA_ENV numpy
mkdir $BUILD_DIRECTORY/build && cd $BUILD_DIRECTORY/build && cmake .. && make lightgbm -j4 || exit -1
cd $BUILD_DIRECTORY/tests/cpp_test && ../../lightgbm config=train.conf convert_model_language=cpp convert_model=../../src/boosting/gbdt_prediction.cpp && ../../lightgbm config=predict.conf output_result=origin.pred || exit -1
cd $BUILD_DIRECTORY/tests/cpp_tests && ../../lightgbm config=train.conf convert_model_language=cpp convert_model=../../src/boosting/gbdt_prediction.cpp && ../../lightgbm config=predict.conf output_result=origin.pred || exit -1
cd $BUILD_DIRECTORY/build && make lightgbm -j4 || exit -1
cd $BUILD_DIRECTORY/tests/cpp_test && ../../lightgbm config=predict.conf output_result=ifelse.pred && python test.py || exit -1
cd $BUILD_DIRECTORY/tests/cpp_tests && ../../lightgbm config=predict.conf output_result=ifelse.pred && python test.py || exit -1
exit 0
fi

Expand All @@ -97,13 +105,7 @@ if [[ $TASK == "swig" ]]; then
fi

conda install -q -y -n $CONDA_ENV cloudpickle dask distributed joblib matplotlib numpy pandas psutil pytest scikit-learn scipy

# graphviz must come from conda-forge to avoid this on some linux distros:
# https://github.com/conda-forge/graphviz-feedstock/issues/18
conda install -q -y \
-n $CONDA_ENV \
-c conda-forge \
python-graphviz
pip install graphviz # python-graphviz from Anaconda is not allowed to be installed with Python 3.9

if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "clang" ]]; then
# fix "OMP: Error #15: Initializing libiomp5.dylib, but found libomp.dylib already initialized." (OpenMP library conflict due to conda's MKL)
Expand Down
15 changes: 1 addition & 14 deletions .ci/test_r_package.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,19 +7,6 @@ mkdir -p $R_LIB_PATH
export R_LIBS=$R_LIB_PATH
export PATH="$R_LIB_PATH/R/bin:$PATH"

# hack to get around this:
# https://stat.ethz.ch/pipermail/r-package-devel/2020q3/005930.html
export _R_CHECK_SYSTEM_CLOCK_=0

# ignore R CMD CHECK NOTE checking how long it has
# been since the last submission
export _R_CHECK_CRAN_INCOMING_REMOTE_=0

# CRAN ignores the "installed size is too large" NOTE,
# so our CI can too. Setting to a large value here just
# to catch extreme problems
export _R_CHECK_PKG_SIZES_THRESHOLD_=100

# don't fail builds for long-running examples unless they're very long.
# See https://github.com/microsoft/LightGBM/issues/4049#issuecomment-793412254.
if [[ $R_BUILD_TYPE != "cran" ]]; then
Expand Down Expand Up @@ -113,7 +100,7 @@ if [[ $OS_NAME == "macos" ]]; then
packages+=", type = 'binary'"
compile_from_source="never"
fi
Rscript --vanilla -e "options(install.packages.compile.from.source = '${compile_from_source}'); install.packages(${packages}, repos = '${CRAN_MIRROR}', lib = '${R_LIB_PATH}', dependencies = c('Depends', 'Imports', 'LinkingTo'))" || exit -1
Rscript --vanilla -e "options(install.packages.compile.from.source = '${compile_from_source}'); install.packages(${packages}, repos = '${CRAN_MIRROR}', lib = '${R_LIB_PATH}', dependencies = c('Depends', 'Imports', 'LinkingTo'), Ncpus = parallel::detectCores())" || exit -1

cd ${BUILD_DIRECTORY}

Expand Down
15 changes: 1 addition & 14 deletions .ci/test_r_package_windows.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -79,19 +79,6 @@ $env:CRAN_MIRROR = "https://cloud.r-project.org/"
$env:CTAN_MIRROR = "https://ctan.math.illinois.edu/systems/win32/miktex"
$env:CTAN_PACKAGE_ARCHIVE = "$env:CTAN_MIRROR/tm/packages/"

# hack to get around this:
# https://stat.ethz.ch/pipermail/r-package-devel/2020q3/005930.html
$env:_R_CHECK_SYSTEM_CLOCK_ = 0

# ignore R CMD CHECK NOTE checking how long it has
# been since the last submission
$env:_R_CHECK_CRAN_INCOMING_REMOTE_ = 0

# CRAN ignores the "installed size is too large" NOTE,
# so our CI can too. Setting to a large value here just
# to catch extreme problems
$env:_R_CHECK_PKG_SIZES_THRESHOLD_ = 100

# don't fail builds for long-running examples unless they're very long.
# See https://github.com/microsoft/LightGBM/issues/4049#issuecomment-793412254.
if ($env:R_BUILD_TYPE -ne "cran") {
Expand Down Expand Up @@ -123,7 +110,7 @@ Write-Output "Done installing Rtools"

Write-Output "Installing dependencies"
$packages = "c('data.table', 'jsonlite', 'Matrix', 'processx', 'R6', 'testthat'), dependencies = c('Imports', 'Depends', 'LinkingTo')"
Run-R-Code-Redirect-Stderr "options(install.packages.check.source = 'no'); install.packages($packages, repos = '$env:CRAN_MIRROR', type = 'binary', lib = '$env:R_LIB_PATH')" ; Check-Output $?
Run-R-Code-Redirect-Stderr "options(install.packages.check.source = 'no'); install.packages($packages, repos = '$env:CRAN_MIRROR', type = 'binary', lib = '$env:R_LIB_PATH', Ncpus = parallel::detectCores())" ; Check-Output $?

# MiKTeX and pandoc can be skipped on non-MinGW builds, since we don't
# build the package documentation for those.
Expand Down
12 changes: 10 additions & 2 deletions .ci/test_windows.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ function Check-Output {
}
}

# unify environment variable for Azure devops and AppVeyor
# unify environment variable for Azure DevOps and AppVeyor
if (Test-Path env:APPVEYOR) {
$env:APPVEYOR = "true"
}
Expand All @@ -16,6 +16,14 @@ if ($env:TASK -eq "r-package") {
Exit 0
}

if ($env:TASK -eq "cpp-tests") {
mkdir $env:BUILD_SOURCESDIRECTORY/build; cd $env:BUILD_SOURCESDIRECTORY/build
cmake -DBUILD_CPP_TEST=ON -DUSE_OPENMP=OFF -A x64 ..
cmake --build . --target testlightgbm --config Debug ; Check-Output $?
Start-Process -FilePath "./../Debug/testlightgbm.exe" -NoNewWindow -Wait ; Check-Output $?
Exit 0
}

# setup for Python
conda init powershell
conda activate
Expand Down Expand Up @@ -58,7 +66,7 @@ elseif ($env:TASK -eq "sdist") {
}
elseif ($env:TASK -eq "bdist") {
# Import the Chocolatey profile module so that the RefreshEnv command
# invoked below properly updates the current PowerShell session enviroment.
# invoked below properly updates the current PowerShell session environment.
$module = "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
Import-Module "$module" ; Check-Output $?
RefreshEnv
Expand Down
2 changes: 1 addition & 1 deletion .github/CODEOWNERS
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ include/ @guolinke @chivee @btrotta @shiyu1994
src/ @guolinke @chivee @btrotta @shiyu1994
CMakeLists.txt @guolinke @chivee @Laurae2 @jameslamb @wxchan @henry0312 @StrikerRUS @huanzhang12 @btrotta @shiyu1994
tests/c_api_test/ @guolinke @chivee @btrotta @shiyu1994
tests/cpp_test/ @guolinke @chivee @btrotta @shiyu1994
tests/cpp_tests/ @guolinke @chivee @btrotta @shiyu1994
tests/data/ @guolinke @chivee @btrotta @shiyu1994
windows/ @guolinke @chivee @btrotta @StrikerRUS @shiyu1994

Expand Down
35 changes: 0 additions & 35 deletions .github/lock.yml

This file was deleted.

96 changes: 96 additions & 0 deletions .github/workflows/cuda.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
name: CUDA Version

on:
push:
branches:
- master
pull_request:
branches:
- master

env:
github_actions: 'true'
os_name: linux
task: cuda
conda_env: test-env

jobs:
test:
name: cuda ${{ matrix.cuda_version }} ${{ matrix.method }} (linux, ${{ matrix.compiler }}, Python ${{ matrix.python_version }})
runs-on: [self-hosted, linux]
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
include:
- method: source
compiler: gcc
python_version: 3.7
cuda_version: "11.2.2"
- method: pip
compiler: clang
python_version: 3.8
cuda_version: "10.0"
- method: wheel
compiler: gcc
python_version: 3.9
cuda_version: "9.0"
steps:
- name: Setup or update software on host machine
run: |
sudo apt-get update
sudo apt-get install --no-install-recommends -y \
apt-transport-https \
ca-certificates \
curl \
git \
gnupg-agent \
software-properties-common
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -y
curl -sL https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
curl -sL https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list
sudo apt-get update
sudo apt-get install --no-install-recommends -y \
containerd.io \
docker-ce \
docker-ce-cli \
nvidia-docker2
sudo chmod a+rw /var/run/docker.sock
sudo systemctl restart docker
- name: Remove old folder with repository
run: sudo rm -rf $GITHUB_WORKSPACE
- name: Checkout repository
uses: actions/checkout@v1
with:
fetch-depth: 5
submodules: true
- name: Setup and run tests
run: |
export ROOT_DOCKER_FOLDER=/LightGBM
cat > docker.env <<EOF
GITHUB_ACTIONS=${{ env.github_actions }}
OS_NAME=${{ env.os_name }}
COMPILER=${{ matrix.compiler }}
TASK=${{ env.task }}
METHOD=${{ matrix.method }}
CONDA_ENV=${{ env.conda_env }}
PYTHON_VERSION=${{ matrix.python_version }}
BUILD_DIRECTORY=$ROOT_DOCKER_FOLDER
LGB_VER=$(head -n 1 VERSION.txt)
EOF
cat > docker-script.sh <<EOF
export CONDA=\$HOME/miniconda
export PATH=\$CONDA/bin:\$PATH
nvidia-smi
$ROOT_DOCKER_FOLDER/.ci/setup.sh || exit -1
$ROOT_DOCKER_FOLDER/.ci/test.sh || exit -1
EOF
docker run --env-file docker.env -v "$GITHUB_WORKSPACE":"$ROOT_DOCKER_FOLDER" --rm --gpus all "nvcr.io/nvidia/cuda:${{ matrix.cuda_version }}-devel" /bin/bash $ROOT_DOCKER_FOLDER/docker-script.sh
all-successful:
# https://git.luolix.topmunity/t/is-it-possible-to-require-all-github-actions-tasks-to-pass-without-enumerating-them/117957/4?u=graingert
runs-on: ubuntu-latest
needs: [test]
steps:
- name: Note that all tests succeeded
run: echo "🎉"
Loading

0 comments on commit 23521a5

Please sign in to comment.