From c836bc480fbe6a426e25b561e236944d52a69bce Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 10 Jan 2023 22:48:57 -0600 Subject: [PATCH 1/5] [ci] fix brew errors in R macOS jobs (fixes #5670) --- .appveyor.yml | 44 --- .ci/test_r_package.sh | 9 +- .github/workflows/cuda.yml | 123 -------- .github/workflows/python_package.yml | 87 ------ .vsts-ci.yml | 400 --------------------------- 5 files changed, 6 insertions(+), 657 deletions(-) delete mode 100644 .appveyor.yml delete mode 100644 .github/workflows/cuda.yml delete mode 100644 .github/workflows/python_package.yml delete mode 100644 .vsts-ci.yml diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index 2d279b0f33e3..000000000000 --- a/.appveyor.yml +++ /dev/null @@ -1,44 +0,0 @@ -version: 3.3.3.99.{build} - -image: Visual Studio 2015 -platform: x64 -configuration: # a trick to construct a build matrix with multiple Python versions - - '3.7' - -# only build pull requests and -# commits to 'master' or any branch starting with 'release' -branches: - only: - - master - - /^release/ - -environment: - matrix: - - COMPILER: MSVC - TASK: python - - COMPILER: MINGW - TASK: python - -clone_depth: 5 - -install: - - git submodule update --init --recursive # get `external_libs` folder - - set PATH=%PATH:C:\Program Files\Git\usr\bin;=% # delete sh.exe from PATH (mingw32-make fix) - - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% - - set PYTHON_VERSION=%CONFIGURATION% - - set CONDA_ENV="test-env" - - ps: | - $env:MINICONDA = "C:\Miniconda3-x64" - $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" - $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER" - $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim() - -build: false - -test_script: - - conda config --remove channels defaults - - conda config --add channels nodefaults - - conda config --add channels conda-forge - - conda config --set channel_priority strict - - conda init powershell - - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test_windows.ps1 diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index b7af507a87b9..1cccd6a5b341 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -81,10 +81,13 @@ if [[ $OS_NAME == "macos" ]]; then if [[ $R_BUILD_TYPE == "cran" ]]; then brew install automake || exit -1 fi - brew install \ - checkbashisms \ - qpdf || exit -1 + echo "--- installing checkbashisms ---" + brew install checkbashisms || exit -1 + echo "--- installing qpdf ---" + brew install qpdf || exit -1 + echo "--- installing basictex ---" brew install --cask basictex || exit -1 + echo "--- tlmgr stuff ---" export PATH="/Library/TeX/texbin:$PATH" sudo tlmgr --verify-repo=none update --self || exit -1 sudo tlmgr --verify-repo=none install inconsolata helvetic rsfs || exit -1 diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml deleted file mode 100644 index 946f548784a6..000000000000 --- a/.github/workflows/cuda.yml +++ /dev/null @@ -1,123 +0,0 @@ -name: CUDA Version - -on: - push: - branches: - - master - pull_request: - branches: - - master - - release/* - -# automatically cancel in-progress builds if another commit is pushed -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - github_actions: 'true' - os_name: linux - conda_env: test-env - -jobs: - test: - name: ${{ matrix.task }} ${{ matrix.cuda_version }} ${{ matrix.method }} (linux, ${{ matrix.compiler }}, Python ${{ matrix.python_version }}) - runs-on: [self-hosted, linux] - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - include: - - method: source - compiler: gcc - python_version: "3.8" - cuda_version: "11.7.1" - task: cuda - - method: pip - compiler: clang - python_version: "3.9" - cuda_version: "10.0" - task: cuda - - method: wheel - compiler: gcc - python_version: "3.10" - cuda_version: "9.0" - task: cuda - - method: source - compiler: gcc - python_version: "3.8" - cuda_version: "11.7.1" - task: cuda_exp - - method: pip - compiler: clang - python_version: "3.9" - cuda_version: "10.0" - task: cuda_exp - steps: - - name: Setup or update software on host machine - run: | - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - apt-transport-https \ - ca-certificates \ - curl \ - git \ - gnupg-agent \ - lsb-release \ - software-properties-common - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -y - curl -sL https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - - curl -sL https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - containerd.io \ - docker-ce \ - docker-ce-cli \ - nvidia-docker2 - sudo chmod a+rw /var/run/docker.sock - sudo systemctl restart docker - - name: Remove old folder with repository - run: sudo rm -rf $GITHUB_WORKSPACE - - name: Checkout repository - uses: actions/checkout@v1 - with: - fetch-depth: 5 - submodules: true - - name: Setup and run tests - run: | - export ROOT_DOCKER_FOLDER=/LightGBM - cat > docker.env < docker-script.sh < '$(Build.ArtifactStagingDirectory)/commit.txt' - displayName: 'Add commit hash to artifacts archive' - - bash: $(Build.SourcesDirectory)/.ci/setup.sh - displayName: Setup - - bash: $(Build.SourcesDirectory)/.ci/test.sh - displayName: Test - - task: PublishBuildArtifacts@1 - condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: PackageAssets - artifactType: container -########################################### -- job: Linux_latest -########################################### - variables: - COMPILER: clang - DEBIAN_FRONTEND: 'noninteractive' - IN_UBUNTU_BASE_CONTAINER: 'true' - OS_NAME: 'linux' - SETUP_CONDA: 'true' - pool: sh-ubuntu - container: ubuntu-latest - strategy: - matrix: - regular: - TASK: regular - sdist: - TASK: sdist - bdist: - TASK: bdist - PYTHON_VERSION: '3.8' - inference: - TASK: if-else - mpi_source: - TASK: mpi - METHOD: source - mpi_pip: - TASK: mpi - METHOD: pip - PYTHON_VERSION: '3.9' - mpi_wheel: - TASK: mpi - METHOD: wheel - PYTHON_VERSION: '3.7' - gpu_source: - TASK: gpu - METHOD: source - PYTHON_VERSION: '3.9' - gpu_pip: - TASK: gpu - METHOD: pip - PYTHON_VERSION: '3.8' - gpu_wheel: - TASK: gpu - METHOD: wheel - PYTHON_VERSION: '3.7' - cpp_tests: - TASK: cpp-tests - METHOD: with-sanitizers - steps: - - script: | - echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" - echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" - CONDA=$HOME/miniforge - echo "##vso[task.setvariable variable=CONDA]$CONDA" - echo "##vso[task.prependpath]$CONDA/bin" - displayName: 'Set variables' - # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301 - - script: | - /tmp/docker exec -t -u 0 ci-container \ - sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo" - displayName: 'Install sudo' - - bash: $(Build.SourcesDirectory)/.ci/setup.sh - displayName: Setup - - bash: $(Build.SourcesDirectory)/.ci/test.sh - displayName: Test -########################################### -- job: QEMU_multiarch -########################################### - variables: - COMPILER: gcc - OS_NAME: 'linux' - PRODUCES_ARTIFACTS: 'true' - pool: - vmImage: ubuntu-22.04 - timeoutInMinutes: 180 - strategy: - matrix: - bdist: - TASK: bdist - ARCH: aarch64 - steps: - - script: | - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - binfmt-support \ - qemu \ - qemu-user \ - qemu-user-static - displayName: 'Install QEMU' - - script: | - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - displayName: 'Enable Docker multi-architecture support' - - script: | - export ROOT_DOCKER_FOLDER=/LightGBM - cat > docker.env < docker-script.sh < Date: Tue, 10 Jan 2023 23:06:47 -0600 Subject: [PATCH 2/5] remove 'brew update' --- .ci/test_r_package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index 1cccd6a5b341..dd900d12167c 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -77,7 +77,7 @@ fi # Installing R precompiled for Mac OS 10.11 or higher if [[ $OS_NAME == "macos" ]]; then - brew update-reset && brew update + brew update-reset if [[ $R_BUILD_TYPE == "cran" ]]; then brew install automake || exit -1 fi From ccb3b3935fc7c498614f74b8955106008f4d6a8f Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 10 Jan 2023 23:21:57 -0600 Subject: [PATCH 3/5] try removing 2to3 links --- .ci/test_r_package.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index dd900d12167c..137578f701a0 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -77,6 +77,7 @@ fi # Installing R precompiled for Mac OS 10.11 or higher if [[ $OS_NAME == "macos" ]]; then + rm -f /usr/local/bin/2to3-3.* brew update-reset if [[ $R_BUILD_TYPE == "cran" ]]; then brew install automake || exit -1 From c08170ce0e1f4fbec12cf09386421c10af26b973 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 11 Jan 2023 20:34:16 -0600 Subject: [PATCH 4/5] remove reset --- .ci/test_r_package.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index 137578f701a0..563b80cfcd00 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -77,8 +77,8 @@ fi # Installing R precompiled for Mac OS 10.11 or higher if [[ $OS_NAME == "macos" ]]; then - rm -f /usr/local/bin/2to3-3.* - brew update-reset + # rm -f /usr/local/bin/2to3* + # rm -f /usr/local/bin/idle3.* if [[ $R_BUILD_TYPE == "cran" ]]; then brew install automake || exit -1 fi From ebdce8720489513aa41a61aa46b1d312cd6a8fc2 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 11 Jan 2023 20:57:11 -0600 Subject: [PATCH 5/5] add all CI back --- .appveyor.yml | 44 +++ .ci/test_r_package.sh | 11 +- .github/workflows/cuda.yml | 123 ++++++++ .github/workflows/python_package.yml | 87 ++++++ .vsts-ci.yml | 400 +++++++++++++++++++++++++++ 5 files changed, 657 insertions(+), 8 deletions(-) create mode 100644 .appveyor.yml create mode 100644 .github/workflows/cuda.yml create mode 100644 .github/workflows/python_package.yml create mode 100644 .vsts-ci.yml diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 000000000000..2d279b0f33e3 --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,44 @@ +version: 3.3.3.99.{build} + +image: Visual Studio 2015 +platform: x64 +configuration: # a trick to construct a build matrix with multiple Python versions + - '3.7' + +# only build pull requests and +# commits to 'master' or any branch starting with 'release' +branches: + only: + - master + - /^release/ + +environment: + matrix: + - COMPILER: MSVC + TASK: python + - COMPILER: MINGW + TASK: python + +clone_depth: 5 + +install: + - git submodule update --init --recursive # get `external_libs` folder + - set PATH=%PATH:C:\Program Files\Git\usr\bin;=% # delete sh.exe from PATH (mingw32-make fix) + - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% + - set PYTHON_VERSION=%CONFIGURATION% + - set CONDA_ENV="test-env" + - ps: | + $env:MINICONDA = "C:\Miniconda3-x64" + $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" + $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER" + $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim() + +build: false + +test_script: + - conda config --remove channels defaults + - conda config --add channels nodefaults + - conda config --add channels conda-forge + - conda config --set channel_priority strict + - conda init powershell + - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test_windows.ps1 diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index 563b80cfcd00..dbc76f061114 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -77,18 +77,13 @@ fi # Installing R precompiled for Mac OS 10.11 or higher if [[ $OS_NAME == "macos" ]]; then - # rm -f /usr/local/bin/2to3* - # rm -f /usr/local/bin/idle3.* if [[ $R_BUILD_TYPE == "cran" ]]; then brew install automake || exit -1 fi - echo "--- installing checkbashisms ---" - brew install checkbashisms || exit -1 - echo "--- installing qpdf ---" - brew install qpdf || exit -1 - echo "--- installing basictex ---" + brew install \ + checkbashisms \ + qpdf || exit -1 brew install --cask basictex || exit -1 - echo "--- tlmgr stuff ---" export PATH="/Library/TeX/texbin:$PATH" sudo tlmgr --verify-repo=none update --self || exit -1 sudo tlmgr --verify-repo=none install inconsolata helvetic rsfs || exit -1 diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml new file mode 100644 index 000000000000..946f548784a6 --- /dev/null +++ b/.github/workflows/cuda.yml @@ -0,0 +1,123 @@ +name: CUDA Version + +on: + push: + branches: + - master + pull_request: + branches: + - master + - release/* + +# automatically cancel in-progress builds if another commit is pushed +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + github_actions: 'true' + os_name: linux + conda_env: test-env + +jobs: + test: + name: ${{ matrix.task }} ${{ matrix.cuda_version }} ${{ matrix.method }} (linux, ${{ matrix.compiler }}, Python ${{ matrix.python_version }}) + runs-on: [self-hosted, linux] + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - method: source + compiler: gcc + python_version: "3.8" + cuda_version: "11.7.1" + task: cuda + - method: pip + compiler: clang + python_version: "3.9" + cuda_version: "10.0" + task: cuda + - method: wheel + compiler: gcc + python_version: "3.10" + cuda_version: "9.0" + task: cuda + - method: source + compiler: gcc + python_version: "3.8" + cuda_version: "11.7.1" + task: cuda_exp + - method: pip + compiler: clang + python_version: "3.9" + cuda_version: "10.0" + task: cuda_exp + steps: + - name: Setup or update software on host machine + run: | + sudo apt-get update + sudo apt-get install --no-install-recommends -y \ + apt-transport-https \ + ca-certificates \ + curl \ + git \ + gnupg-agent \ + lsb-release \ + software-properties-common + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -y + curl -sL https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - + curl -sL https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list + sudo apt-get update + sudo apt-get install --no-install-recommends -y \ + containerd.io \ + docker-ce \ + docker-ce-cli \ + nvidia-docker2 + sudo chmod a+rw /var/run/docker.sock + sudo systemctl restart docker + - name: Remove old folder with repository + run: sudo rm -rf $GITHUB_WORKSPACE + - name: Checkout repository + uses: actions/checkout@v1 + with: + fetch-depth: 5 + submodules: true + - name: Setup and run tests + run: | + export ROOT_DOCKER_FOLDER=/LightGBM + cat > docker.env < docker-script.sh < '$(Build.ArtifactStagingDirectory)/commit.txt' + displayName: 'Add commit hash to artifacts archive' + - bash: $(Build.SourcesDirectory)/.ci/setup.sh + displayName: Setup + - bash: $(Build.SourcesDirectory)/.ci/test.sh + displayName: Test + - task: PublishBuildArtifacts@1 + condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: PackageAssets + artifactType: container +########################################### +- job: Linux_latest +########################################### + variables: + COMPILER: clang + DEBIAN_FRONTEND: 'noninteractive' + IN_UBUNTU_BASE_CONTAINER: 'true' + OS_NAME: 'linux' + SETUP_CONDA: 'true' + pool: sh-ubuntu + container: ubuntu-latest + strategy: + matrix: + regular: + TASK: regular + sdist: + TASK: sdist + bdist: + TASK: bdist + PYTHON_VERSION: '3.8' + inference: + TASK: if-else + mpi_source: + TASK: mpi + METHOD: source + mpi_pip: + TASK: mpi + METHOD: pip + PYTHON_VERSION: '3.9' + mpi_wheel: + TASK: mpi + METHOD: wheel + PYTHON_VERSION: '3.7' + gpu_source: + TASK: gpu + METHOD: source + PYTHON_VERSION: '3.9' + gpu_pip: + TASK: gpu + METHOD: pip + PYTHON_VERSION: '3.8' + gpu_wheel: + TASK: gpu + METHOD: wheel + PYTHON_VERSION: '3.7' + cpp_tests: + TASK: cpp-tests + METHOD: with-sanitizers + steps: + - script: | + echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" + echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" + CONDA=$HOME/miniforge + echo "##vso[task.setvariable variable=CONDA]$CONDA" + echo "##vso[task.prependpath]$CONDA/bin" + displayName: 'Set variables' + # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301 + - script: | + /tmp/docker exec -t -u 0 ci-container \ + sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo" + displayName: 'Install sudo' + - bash: $(Build.SourcesDirectory)/.ci/setup.sh + displayName: Setup + - bash: $(Build.SourcesDirectory)/.ci/test.sh + displayName: Test +########################################### +- job: QEMU_multiarch +########################################### + variables: + COMPILER: gcc + OS_NAME: 'linux' + PRODUCES_ARTIFACTS: 'true' + pool: + vmImage: ubuntu-22.04 + timeoutInMinutes: 180 + strategy: + matrix: + bdist: + TASK: bdist + ARCH: aarch64 + steps: + - script: | + sudo apt-get update + sudo apt-get install --no-install-recommends -y \ + binfmt-support \ + qemu \ + qemu-user \ + qemu-user-static + displayName: 'Install QEMU' + - script: | + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + displayName: 'Enable Docker multi-architecture support' + - script: | + export ROOT_DOCKER_FOLDER=/LightGBM + cat > docker.env < docker-script.sh <