From c02e83f204fcfafbc066b387f7bcc962e695f02c Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 14 Oct 2021 00:18:57 -0500 Subject: [PATCH] Revert "comment out CI" This reverts commit 899fbb4574b99a8125b28761425e3e821bfdfef1. --- .appveyor.yml | 43 +++ .github/workflows/cuda.yml | 103 +++++++ .github/workflows/python_package.yml | 79 ++++++ .github/workflows/r_package.yml | 358 +++++++++++------------ .github/workflows/static_analysis.yml | 85 ++++++ .vsts-ci.yml | 390 ++++++++++++++++++++++++++ 6 files changed, 879 insertions(+), 179 deletions(-) create mode 100644 .appveyor.yml create mode 100644 .github/workflows/cuda.yml create mode 100644 .github/workflows/python_package.yml create mode 100644 .github/workflows/static_analysis.yml create mode 100644 .vsts-ci.yml diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 000000000000..229a495588a8 --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,43 @@ +version: 3.3.0.99.{build} + +image: Visual Studio 2015 +platform: x64 +configuration: # a trick to construct a build matrix with multiple Python versions + - 3.7 + +# only build pull requests and +# commits to 'master' +branches: + only: + - master + +environment: + matrix: + - COMPILER: MSVC + TASK: python + - COMPILER: MINGW + TASK: python + +clone_depth: 5 + +install: + - git submodule update --init --recursive # get `external_libs` folder + - set PATH=%PATH:C:\Program Files\Git\usr\bin;=% # delete sh.exe from PATH (mingw32-make fix) + - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% + - set PYTHON_VERSION=%CONFIGURATION% + - set CONDA_ENV="test-env" + - ps: | + switch ($env:PYTHON_VERSION) { + "3.6" {$env:MINICONDA = "C:\Miniconda36-x64"} + "3.7" {$env:MINICONDA = "C:\Miniconda37-x64"} + default {$env:MINICONDA = "C:\Miniconda37-x64"} + } + $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" + $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER" + $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim() + +build: false + +test_script: + - conda init powershell + - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test_windows.ps1 diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml new file mode 100644 index 000000000000..822ea2d44b82 --- /dev/null +++ b/.github/workflows/cuda.yml @@ -0,0 +1,103 @@ +name: CUDA Version + +on: + push: + branches: + - master + pull_request: + branches: + - master + +env: + github_actions: 'true' + os_name: linux + task: cuda + conda_env: test-env + +jobs: + test: + name: cuda ${{ matrix.cuda_version }} ${{ matrix.method }} (linux, ${{ matrix.compiler }}, Python ${{ matrix.python_version }}) + runs-on: [self-hosted, linux] + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - method: source + compiler: gcc + python_version: 3.7 + cuda_version: "11.4.2" + - method: pip + compiler: clang + python_version: 3.8 + cuda_version: "10.0" + - method: wheel + compiler: gcc + python_version: 3.9 + cuda_version: "9.0" + steps: + - name: Setup or update software on host machine + run: | + sudo apt-get update + sudo apt-get install --no-install-recommends -y \ + apt-transport-https \ + ca-certificates \ + curl \ + git \ + gnupg-agent \ + lsb-release \ + software-properties-common + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -y + curl -sL https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - + curl -sL https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list + sudo apt-get update + sudo apt-get install --no-install-recommends -y \ + containerd.io \ + docker-ce \ + docker-ce-cli \ + nvidia-docker2 + sudo chmod a+rw /var/run/docker.sock + sudo systemctl restart docker + - name: Remove old folder with repository + run: sudo rm -rf $GITHUB_WORKSPACE + - name: Checkout repository + uses: actions/checkout@v1 + with: + fetch-depth: 5 + submodules: true + - name: Setup and run tests + run: | + export ROOT_DOCKER_FOLDER=/LightGBM + cat > docker.env < docker-script.sh <> tests.log 2>&1 || exit_code=1 cat ./tests.log exit ${exit_code} - # test-r-debian-clang: - # name: r-package (debian, R-devel, clang) - # timeout-minutes: 60 - # runs-on: ubuntu-latest - # container: rhub/debian-clang-devel - # steps: - # - name: Install Git before checkout - # shell: bash - # run: | - # apt-get update --allow-releaseinfo-change - # apt-get install --no-install-recommends -y git - # - name: Checkout repository - # uses: actions/checkout@v2.3.4 - # with: - # fetch-depth: 5 - # submodules: true - # - name: Install packages and run tests - # shell: bash - # run: | - # export PATH=/opt/R-devel/bin/:${PATH} - # Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'Matrix', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" - # sh build-cran-package.sh - # R CMD check --as-cran --run-donttest lightgbm_*.tar.gz || exit -1 - # if grep -q -E "NOTE|WARNING|ERROR" lightgbm.Rcheck/00check.log; then - # echo "NOTEs, WARNINGs, or ERRORs have been found by R CMD check" - # exit -1 - # fi + test-r-debian-clang: + name: r-package (debian, R-devel, clang) + timeout-minutes: 60 + runs-on: ubuntu-latest + container: rhub/debian-clang-devel + steps: + - name: Install Git before checkout + shell: bash + run: | + apt-get update --allow-releaseinfo-change + apt-get install --no-install-recommends -y git + - name: Checkout repository + uses: actions/checkout@v2.3.4 + with: + fetch-depth: 5 + submodules: true + - name: Install packages and run tests + shell: bash + run: | + export PATH=/opt/R-devel/bin/:${PATH} + Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'Matrix', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" + sh build-cran-package.sh + R CMD check --as-cran --run-donttest lightgbm_*.tar.gz || exit -1 + if grep -q -E "NOTE|WARNING|ERROR" lightgbm.Rcheck/00check.log; then + echo "NOTEs, WARNINGs, or ERRORs have been found by R CMD check" + exit -1 + fi all-successful: # https://github.community/t/is-it-possible-to-require-all-github-actions-tasks-to-pass-without-enumerating-them/117957/4?u=graingert runs-on: ubuntu-latest - needs: [test-r-sanitizers] + needs: [test, test-r-sanitizers, test-r-debian-clang] steps: - name: Note that all tests succeeded run: echo "🎉" diff --git a/.github/workflows/static_analysis.yml b/.github/workflows/static_analysis.yml new file mode 100644 index 000000000000..cdc917903f46 --- /dev/null +++ b/.github/workflows/static_analysis.yml @@ -0,0 +1,85 @@ +# contains non-functional tests, like checks on docs +# and code style +name: Static Analysis + +on: + push: + branches: + - master + pull_request: + branches: + - master + +env: + COMPILER: 'gcc' + CONDA_ENV: test-env + GITHUB_ACTIONS: 'true' + OS_NAME: 'linux' + PYTHON_VERSION: 3.9 + +jobs: + test: + name: ${{ matrix.task }} + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - task: lint + - task: check-docs + steps: + - name: Checkout repository + uses: actions/checkout@v2.3.4 + with: + fetch-depth: 5 + submodules: false + - name: Setup and run tests + shell: bash + run: | + export TASK="${{ matrix.task }}" + export BUILD_DIRECTORY="$GITHUB_WORKSPACE" + export CONDA=${HOME}/miniconda + export PATH=${CONDA}/bin:$HOME/.local/bin:${PATH} + $GITHUB_WORKSPACE/.ci/setup.sh || exit -1 + $GITHUB_WORKSPACE/.ci/test.sh || exit -1 + r-check-docs: + name: r-package-check-docs + timeout-minutes: 60 + runs-on: ubuntu-latest + container: rocker/verse + steps: + - name: Checkout repository + uses: actions/checkout@v2.3.4 + with: + fetch-depth: 5 + submodules: true + - name: Install packages + shell: bash + run: | + Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'Matrix', 'roxygen2', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" + sh build-cran-package.sh || exit -1 + R CMD INSTALL --with-keep.source lightgbm_*.tar.gz || exit -1 + - name: Test documentation + shell: bash --noprofile --norc {0} + run: | + Rscript --vanilla -e "roxygen2::roxygenize('R-package/', load = 'installed')" || exit -1 + num_doc_files_changed=$( + git diff --name-only | grep --count -E "\.Rd|NAMESPACE" + ) + if [[ ${num_doc_files_changed} -gt 0 ]]; then + echo "Some R documentation files have changed. Please re-generate them and commit those changes." + echo "" + echo " sh build-cran-package.sh" + echo " R CMD INSTALL --with-keep.source lightgbm_*.tar.gz" + echo " Rscript -e \"roxygen2::roxygenize('R-package/', load = 'installed')\"" + echo "" + exit -1 + fi + all-successful: + # https://github.community/t/is-it-possible-to-require-all-github-actions-tasks-to-pass-without-enumerating-them/117957/4?u=graingert + runs-on: ubuntu-latest + needs: [test, r-check-docs] + steps: + - name: Note that all tests succeeded + run: echo "🎉" diff --git a/.vsts-ci.yml b/.vsts-ci.yml new file mode 100644 index 000000000000..059ebf9d35b4 --- /dev/null +++ b/.vsts-ci.yml @@ -0,0 +1,390 @@ +trigger: + branches: + include: + - master + tags: + include: + - v* +pr: +- master +variables: + AZURE: 'true' + PYTHON_VERSION: 3.9 + CONDA_ENV: test-env +resources: + containers: + - container: ubuntu1404 + image: lightgbm/vsts-agent:ubuntu-14.04 + - container: ubuntu-latest + image: 'ubuntu:latest' + options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro" + - container: rbase + image: rocker/r-base +jobs: +########################################### +- job: Linux +########################################### + variables: + COMPILER: gcc + SETUP_CONDA: 'false' + OS_NAME: 'linux' + PRODUCES_ARTIFACTS: 'true' + pool: sh-ubuntu + container: ubuntu1404 + strategy: + matrix: + regular: + TASK: regular + sdist: + TASK: sdist + PYTHON_VERSION: 3.7 + bdist: + TASK: bdist + inference: + TASK: if-else + mpi_source: + TASK: mpi + METHOD: source + PYTHON_VERSION: 3.8 + gpu_source: + TASK: gpu + METHOD: source + swig: + TASK: swig + steps: + - script: | + echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" + echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" + echo "##vso[task.prependpath]$CONDA/bin" + AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK + echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH" + LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH + echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH" + echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors" + displayName: 'Set variables' + - bash: $(Build.SourcesDirectory)/.ci/setup.sh + displayName: Setup + - bash: $(Build.SourcesDirectory)/.ci/test.sh + displayName: Test + - task: PublishBuildArtifacts@1 + condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: PackageAssets + artifactType: container +########################################### +- job: Linux_latest +########################################### + variables: + COMPILER: clang + DEBIAN_FRONTEND: 'noninteractive' + IN_UBUNTU_LATEST_CONTAINER: 'true' + OS_NAME: 'linux' + SETUP_CONDA: 'true' + pool: sh-ubuntu + container: ubuntu-latest + strategy: + matrix: + regular: + TASK: regular + PYTHON_VERSION: 3.6 + sdist: + TASK: sdist + bdist: + TASK: bdist + PYTHON_VERSION: 3.8 + inference: + TASK: if-else + mpi_source: + TASK: mpi + METHOD: source + mpi_pip: + TASK: mpi + METHOD: pip + PYTHON_VERSION: 3.8 + mpi_wheel: + TASK: mpi + METHOD: wheel + PYTHON_VERSION: 3.7 + gpu_source: + TASK: gpu + METHOD: source + gpu_pip: + TASK: gpu + METHOD: pip + PYTHON_VERSION: 3.6 + gpu_wheel: + TASK: gpu + METHOD: wheel + PYTHON_VERSION: 3.7 + cpp_tests: + TASK: cpp-tests + METHOD: with-sanitizers + steps: + - script: | + echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" + echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" + CONDA=$HOME/miniconda + echo "##vso[task.setvariable variable=CONDA]$CONDA" + echo "##vso[task.prependpath]$CONDA/bin" + AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK + echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH" + LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH + echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH" + echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors" + displayName: 'Set variables' + # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301 + - script: | + /tmp/docker exec -t -u 0 ci-container \ + sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo" + displayName: 'Install sudo' + - bash: $(Build.SourcesDirectory)/.ci/setup.sh + displayName: Setup + - bash: $(Build.SourcesDirectory)/.ci/test.sh + displayName: Test +########################################### +- job: QEMU_multiarch +########################################### + variables: + COMPILER: gcc + OS_NAME: 'linux' + PRODUCES_ARTIFACTS: 'true' + pool: + vmImage: ubuntu-latest + timeoutInMinutes: 180 + strategy: + matrix: + bdist: + TASK: bdist + ARCH: aarch64 + steps: + - script: | + sudo apt-get update + sudo apt-get install --no-install-recommends -y \ + binfmt-support \ + qemu \ + qemu-user \ + qemu-user-static + displayName: 'Install QEMU' + - script: | + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + displayName: 'Enable Docker multi-architecture support' + - script: | + export ROOT_DOCKER_FOLDER=/LightGBM + cat > docker.env < docker-script.sh <