[ci] remove Travis (fixes #3519) (#3672)

* [ci] move CI jobs from Travis to Azure DevOps (fixes #3519)

* comment out other CIs to avoid wasting cycles

* try without docker

* add container back

* stop using --user in pip install

* run check-docs and lint without container

* job names

* move more jobs to Azure-hosted Linux pool

* fix PATH for check-docs

* uncomment other CI

* uncomment windows

* remove uses of maxParallel

* try moving macos-latest jobs to GitHub Actions

* fix config

* fix missing conda env

* set Python version

* remove commented-out code

* add more to GitHub Actions

* try to fix GPU

* remove static_analysis to prevent conflicts with #3726

* change workflow name

* try using ubuntu:latest docker

* fix conda

* trying to find where permissions first break

* add workaround for sudo

* please azure please

* image syntax

* more sudo

* noninteractive

* LC_ALL

* more  sudo

* more stuff

* CONDA dir

* paths

* get path

* missing CONDA

* fix path stuff

* more tests

* fix graphviz

* stuff

* more graphviz

* install xorg-libxau

* graphviz works, run more jobs

* stuff

* enable more tests

* uncomment GitHub Actions

* uncomment all other CIs

* Apply suggestions from code review

Co-authored-by: Nikita Titov <nekit94-08@mail.ru>

* add travis.yml to Rbuildignore

* add Rbuildignore rule for fmt

* add libomp for clang builds

* changes from code review

Co-authored-by: Nikita Titov <nekit94-08@mail.ru>

.ci/setup.sh

@@ -19,6 +19,47 @@ if [[ $OS_NAME == "macos" ]]; then
     fi
     wget -q -O conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
 else  # Linux
+    if [[ $IN_UBUNTU_LATEST_CONTAINER == "true" ]]; then
+
+        # fixes error "unable to initialize frontend: Dialog"
+        # https://github.com/moby/moby/issues/27988#issuecomment-462809153
+        echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections
+
+        sudo apt-get update
+        sudo apt-get install -y --no-install-recommends \
+            software-properties-common
+
+        sudo add-apt-repository -y ppa:git-core/ppa
+        sudo apt-get update
+
+        sudo apt-get install -y --no-install-recommends \
+            apt-utils \
+            build-essential \
+            ca-certificates \
+            curl \
+            git \
+            iputils-ping \
+            jq \
+            libcurl4 \
+            libicu66 \
+            libssl1.1 \
+            libunwind8 \
+            locales \
+            netcat \
+            unzip \
+            wget \
+            zip
+
+        export LANG="en_US.UTF-8"
+        export LC_ALL="${LANG}"
+        sudo locale-gen ${LANG}
+        sudo update-locale
+
+        sudo apt-get install -y --no-install-recommends \
+            cmake \
+            clang \
+            libomp-dev
+    fi
     if [[ $TASK == "mpi" ]]; then
         sudo apt-get update
         sudo apt-get install --no-install-recommends -y libopenmpi-dev openmpi-bin

.ci/test.sh

@@ -68,7 +68,15 @@ if [[ $TASK == "if-else" ]]; then
     exit 0
 fi
 
-conda install -q -y -n $CONDA_ENV dask dask-ml distributed joblib matplotlib numpy pandas psutil pytest python-graphviz scikit-learn scipy
+conda install -q -y -n $CONDA_ENV dask dask-ml distributed joblib matplotlib numpy pandas psutil pytest scikit-learn scipy
+
+# graphviz must come from conda-forge to avoid this on some linux distros:
+# https://github.com/conda-forge/graphviz-feedstock/issues/18
+conda install -q -y \
+    -n $CONDA_ENV \
+    -c conda-forge \
+        python-graphviz \
+        xorg-libxau
 
 if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "clang" ]]; then
     # fix "OMP: Error #15: Initializing libiomp5.dylib, but found libomp.dylib already initialized." (OpenMP library conflict due to conda's MKL)
@@ -78,7 +86,7 @@ fi
 if [[ $TASK == "sdist" ]]; then
     cd $BUILD_DIRECTORY/python-package && python setup.py sdist || exit -1
     pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v || exit -1
-    if [[ $AZURE == "true" ]]; then
+    if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
         cp $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz $BUILD_ARTIFACTSTAGINGDIRECTORY
         mkdir $BUILD_DIRECTORY/build && cd $BUILD_DIRECTORY/build
         if [[ $OS_NAME == "macos" ]]; then
@@ -100,12 +108,12 @@ elif [[ $TASK == "bdist" ]]; then
     if [[ $OS_NAME == "macos" ]]; then
         cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --plat-name=macosx --python-tag py3 || exit -1
         mv dist/lightgbm-$LGB_VER-py3-none-macosx.whl dist/lightgbm-$LGB_VER-py3-none-macosx_10_13_x86_64.macosx_10_14_x86_64.macosx_10_15_x86_64.whl
-        if [[ $AZURE == "true" ]]; then
+        if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
             cp dist/lightgbm-$LGB_VER-py3-none-macosx*.whl $BUILD_ARTIFACTSTAGINGDIRECTORY
         fi
     else
         cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --plat-name=manylinux1_x86_64 --python-tag py3 || exit -1
-        if [[ $AZURE == "true" ]]; then
+        if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
             cp dist/lightgbm-$LGB_VER-py3-none-manylinux1_x86_64.whl $BUILD_ARTIFACTSTAGINGDIRECTORY
         fi
     fi
@@ -172,7 +180,7 @@ cd $BUILD_DIRECTORY/python-package && python setup.py install --precompile --use
 pytest $BUILD_DIRECTORY/tests || exit -1
 
 if [[ $TASK == "regular" ]]; then
-    if [[ $AZURE == "true" ]]; then
+    if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
         if [[ $OS_NAME == "macos" ]]; then
             cp $BUILD_DIRECTORY/lib_lightgbm.so $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.dylib
         else

.vsts-ci.yml

@@ -15,13 +15,18 @@ resources:
   containers:
   - container: ubuntu1404
     image: lightgbm/vsts-agent:ubuntu-14.04
+  - container: ubuntu-latest
+    image: 'ubuntu:latest'
+    options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro"
 jobs:
 ###########################################
 - job: Linux
 ###########################################
   variables:
     COMPILER: gcc
     SETUP_CONDA: 'false'
+    OS_NAME: 'linux'
+    PRODUCES_ARTIFACTS: 'true'
   pool: sh-ubuntu
   container: ubuntu1404
   strategy:
@@ -41,11 +46,9 @@ jobs:
       gpu_source:
         TASK: gpu
         METHOD: source
-        PYTHON_VERSION: 3.6
   steps:
   - script: |
       echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
-      echo "##vso[task.setvariable variable=OS_NAME]linux"
       echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
       echo "##vso[task.prependpath]$CONDA/bin"
       AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK
@@ -65,10 +68,78 @@ jobs:
       artifactName: PackageAssets
       artifactType: container
 ###########################################
+- job: Linux_latest
+###########################################
+  variables:
+    COMPILER: clang
+    DEBIAN_FRONTEND: 'noninteractive'
+    IN_UBUNTU_LATEST_CONTAINER: 'true'
+    OS_NAME: 'linux'
+    SETUP_CONDA: 'true'
+  pool: sh-ubuntu
+  container: ubuntu-latest
+  strategy:
+    matrix:
+      regular:
+        TASK: regular
+        PYTHON_VERSION: 3.6
+      sdist:
+        TASK: sdist
+      bdist:
+        TASK: bdist
+      inference:
+        TASK: if-else
+      mpi_source:
+        TASK: mpi
+        METHOD: source
+      mpi_pip:
+        TASK: mpi
+        METHOD: pip
+        PYTHON_VERSION: 3.7
+      mpi_wheel:
+        TASK: mpi
+        METHOD: wheel
+        PYTHON_VERSION: 3.7
+      gpu_source:
+        TASK: gpu
+        METHOD: source
+      gpu_pip:
+        TASK: gpu
+        METHOD: pip
+        PYTHON_VERSION: 3.6
+      gpu_wheel:
+        TASK: gpu
+        METHOD: wheel
+        PYTHON_VERSION: 3.7
+  steps:
+  - script: |
+      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
+      echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
+      CONDA=$HOME/miniconda
+      echo "##vso[task.setvariable variable=CONDA]$CONDA"
+      echo "##vso[task.prependpath]$CONDA/bin"
+      AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK
+      echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH"
+      LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH
+      echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH"
+      echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors"
+    displayName: 'Set variables'
+  # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301
+  - script: |
+      /tmp/docker exec -t -u 0 ci-container \
+      sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
+    displayName: 'Install sudo'
+  - bash: $(Build.SourcesDirectory)/.ci/setup.sh
+    displayName: Setup
+  - bash: $(Build.SourcesDirectory)/.ci/test.sh
+    displayName: Test
+###########################################
 - job: MacOS
 ###########################################
   variables:
     COMPILER: clang
+    OS_NAME: 'macos'
+    PRODUCES_ARTIFACTS: 'true'
   pool:
     vmImage: 'macOS-10.14'
   strategy:
@@ -83,7 +154,6 @@ jobs:
   steps:
   - script: |
       echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
-      echo "##vso[task.setvariable variable=OS_NAME]macos"
       echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
       CONDA=$AGENT_HOMEDIRECTORY/miniconda
       echo "##vso[task.setvariable variable=CONDA]$CONDA"
@@ -139,6 +209,7 @@ jobs:
 ###########################################
   dependsOn:
   - Linux
+  - Linux_latest
   - MacOS
   - Windows
   condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))

R-package/.Rbuildignore

@@ -35,6 +35,7 @@ src/external_libs/fast_double_parser/tests
 src/external_libs/fast_double_parser/.*\.yaml
 src/external_libs/fast_double_parser/.*\.yml
 src/external_libs/fmt/.*\.md
+src/external_libs/fmt/.travis.yml
 src/external_libs/fmt/doc
 src/external_libs/fmt/support/Android\.mk
 src/external_libs/fmt/support/.*\.gradle
@@ -46,4 +47,3 @@ src/external_libs/fmt/support/Vagrantfile
 src/external_libs/fmt/support/.*\.xml
 src/external_libs/fmt/support/.*\.yml
 src/external_libs/fmt/test
-\.travis\.yml

README.md

@@ -8,7 +8,6 @@ Light Gradient Boosting Machine
 [![Static Analysis GitHub Actions Build Status](https://github.com/microsoft/LightGBM/workflows/Static%20Analysis/badge.svg?branch=master)](https://github.com/microsoft/LightGBM/actions)
 [![Azure Pipelines Build Status](https://lightgbm-ci.visualstudio.com/lightgbm-ci/_apis/build/status/Microsoft.LightGBM?branchName=master)](https://lightgbm-ci.visualstudio.com/lightgbm-ci/_build/latest?definitionId=1)
 [![Appveyor Build Status](https://ci.appveyor.com/api/projects/status/1ys5ot401m0fep6l/branch/master?svg=true)](https://ci.appveyor.com/project/guolinke/lightgbm/branch/master)
-[![Travis Build Status](https://travis-ci.org/microsoft/LightGBM.svg?branch=master)](https://travis-ci.org/microsoft/LightGBM)
 [![Documentation Status](https://readthedocs.org/projects/lightgbm/badge/?version=latest)](https://lightgbm.readthedocs.io/)
 [![License](https://img.shields.io/github/license/microsoft/lightgbm.svg)](https://github.com/microsoft/LightGBM/blob/master/LICENSE)
 [![Python Versions](https://img.shields.io/pypi/pyversions/lightgbm.svg?logo=python&logoColor=white)](https://pypi.org/project/lightgbm)