diff --git a/Jenkinsfile b/Jenkinsfile index 96268901ee70..3dd9f281b2cf 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,129 +3,270 @@ // Jenkins pipeline // See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ -import groovy.transform.Field - -/* Unrestricted tasks: tasks that do NOT generate artifacts */ - // Command to run command inside a docker container -def dockerRun = 'tests/ci_build/ci_build.sh' -// Utility functions -@Field -def utils - -def buildMatrix = [ - [ "enabled": true, "os" : "linux", "withGpu": true, "withNccl": true, "withOmp": true, "pythonVersion": "2.7", "cudaVersion": "10.0", "multiGpu": true], - [ "enabled": true, "os" : "linux", "withGpu": true, "withNccl": true, "withOmp": true, "pythonVersion": "2.7", "cudaVersion": "9.2" ], - [ "enabled": true, "os" : "linux", "withGpu": true, "withNccl": true, "withOmp": true, "pythonVersion": "2.7", "cudaVersion": "8.0" ], - [ "enabled": true, "os" : "linux", "withGpu": true, "withNccl": false, "withOmp": true, "pythonVersion": "2.7", "cudaVersion": "8.0" ], -] +dockerRun = 'tests/ci_build/ci_build.sh' pipeline { - // Each stage specify its own agent - agent none + // Each stage specify its own agent + agent none - environment { - DOCKER_CACHE_REPO = '492475357299.dkr.ecr.us-west-2.amazonaws.com' - } + environment { + DOCKER_CACHE_REPO = '492475357299.dkr.ecr.us-west-2.amazonaws.com' + } - // Setup common job properties - options { - ansiColor('xterm') - timestamps() - timeout(time: 120, unit: 'MINUTES') - buildDiscarder(logRotator(numToKeepStr: '10')) - } + // Setup common job properties + options { + ansiColor('xterm') + timestamps() + timeout(time: 120, unit: 'MINUTES') + buildDiscarder(logRotator(numToKeepStr: '10')) + preserveStashes() + } - // Build stages - stages { - stage('Jenkins: Get sources') { - agent { - label 'unrestricted' - } - steps { - script { - utils = load('tests/ci_build/jenkins_tools.Groovy') - utils.checkoutSrcs() - } - stash name: 'srcs', excludes: '.git/' - milestone label: 'Sources ready', ordinal: 1 - } + // Build stages + stages { + stage('Get sources') { + agent { label 'linux && cpu' } + steps { + script { + checkoutSrcs() } - stage('Jenkins: Build & Test') { - steps { - script { - parallel (buildMatrix.findAll{it['enabled']}.collectEntries{ c -> - def buildName = utils.getBuildName(c) - utils.buildFactory(buildName, c, false, this.&buildPlatformCmake) - } + [ "clang-tidy" : { buildClangTidyJob() } ]) - } - } + stash name: 'srcs' + milestone ordinal: 1 + } + } + stage('Formatting Check') { + agent none + steps { + script { + parallel ([ + 'clang-tidy': { ClangTidy() }, + 'lint': { Lint() }, + 'sphinx-doc': { SphinxDoc() }, + 'doxygen': { Doxygen() } + ]) } + milestone ordinal: 2 + } } -} - -/** - * Build platform and test it via cmake. - */ -def buildPlatformCmake(buildName, conf, nodeReq, dockerTarget) { - def opts = utils.cmakeOptions(conf) - // Destination dir for artifacts - def distDir = "dist/${buildName}" - def dockerArgs = "" - if (conf["withGpu"]) { - dockerArgs = "--build-arg CUDA_VERSION=" + conf["cudaVersion"] + stage('Build') { + agent none + steps { + script { + parallel ([ + 'build-cpu': { BuildCPU() }, + 'build-gpu-cuda8.0': { BuildCUDA(cuda_version: '8.0') }, + 'build-gpu-cuda9.2': { BuildCUDA(cuda_version: '9.2') }, + 'build-gpu-cuda10.0': { BuildCUDA(cuda_version: '10.0') }, + 'build-jvm-packages': { BuildJVMPackages(spark_version: '2.4.1') }, + 'build-jvm-doc': { BuildJVMDoc() } + ]) + } + milestone ordinal: 3 + } } - def test_suite = conf["withGpu"] ? (conf["multiGpu"] ? "mgpu" : "gpu") : "cpu" - // Build node - this is returned result - retry(1) { - node(nodeReq) { - unstash name: 'srcs' - echo """ - |===== XGBoost CMake build ===== - | dockerTarget: ${dockerTarget} - | cmakeOpts : ${opts} - |========================= - """.stripMargin('|') - // Invoke command inside docker - sh """ - ${dockerRun} ${dockerTarget} ${dockerArgs} tests/ci_build/build_via_cmake.sh ${opts} - ${dockerRun} ${dockerTarget} ${dockerArgs} tests/ci_build/test_${test_suite}.sh - """ - if (!conf["multiGpu"]) { - sh """ - ${dockerRun} ${dockerTarget} ${dockerArgs} bash -c "cd python-package; rm -f dist/*; python setup.py bdist_wheel --universal" - rm -rf "${distDir}"; mkdir -p "${distDir}/py" - cp xgboost "${distDir}" - cp -r python-package/dist "${distDir}/py" - # Test the wheel for compatibility on a barebones CPU container - ${dockerRun} release ${dockerArgs} bash -c " \ - pip install --user python-package/dist/xgboost-*-none-any.whl && \ - pytest -v --fulltrace -s tests/python" - # Test the wheel for compatibility on CUDA 10.0 container - ${dockerRun} gpu --build-arg CUDA_VERSION=10.0 bash -c " \ - pip install --user python-package/dist/xgboost-*-none-any.whl && \ - pytest -v -s --fulltrace -m '(not mgpu) and (not slow)' tests/python-gpu" - """ - } + stage('Test') { + agent none + steps { + script { + parallel ([ + 'test-python-cpu': { TestPythonCPU() }, + 'test-python-gpu-cuda8.0': { TestPythonGPU(cuda_version: '8.0') }, + 'test-python-gpu-cuda9.2': { TestPythonGPU(cuda_version: '9.2') }, + 'test-python-gpu-cuda10.0': { TestPythonGPU(cuda_version: '10.0') }, + 'test-cpp-gpu': { TestCppGPU(cuda_version: '10.0') }, + 'test-cpp-mgpu': { TestCppGPU(cuda_version: '10.0', multi_gpu: true) }, + 'test-jvm-jdk8': { CrossTestJVMwithJDK(jdk_version: '8') }, + 'test-jvm-jdk11': { CrossTestJVMwithJDK(jdk_version: '11') }, + 'test-r-3.4.4': { TestR(r_version: '3.4.4') }, + 'test-r-3.5.3': { TestR(r_version: '3.5.3') } + ]) } + milestone ordinal: 4 + } } + } } -/** - * Run a clang-tidy job on a GPU machine - */ -def buildClangTidyJob() { - def nodeReq = "linux && gpu && unrestricted" - node(nodeReq) { - unstash name: 'srcs' - echo "Running clang-tidy job..." - // Invoke command inside docker - // Install Google Test and Python yaml - dockerTarget = "clang_tidy" - dockerArgs = "--build-arg CUDA_VERSION=9.2" - sh """ - ${dockerRun} ${dockerTarget} ${dockerArgs} tests/ci_build/clang_tidy.sh - """ +// check out source code from git +def checkoutSrcs() { + retry(5) { + try { + timeout(time: 2, unit: 'MINUTES') { + checkout scm + sh 'git submodule update --init' } + } catch (exc) { + deleteDir() + error "Failed to fetch source codes" + } + } +} + +def ClangTidy() { + node('linux && cpu') { + unstash name: 'srcs' + echo "Running clang-tidy job..." + container_type = "clang_tidy" + docker_binary = "docker" + dockerArgs = "--build-arg CUDA_VERSION=9.2" + sh """ + ${dockerRun} ${container_type} ${docker_binary} ${dockerArgs} tests/ci_build/clang_tidy.sh + """ + deleteDir() + } +} + +def Lint() { + node('linux && cpu') { + unstash name: 'srcs' + echo "Running lint..." + // commented out for now, until another PR to migrate lint to Python 3 gets merged + container_type = "lint" + docker_binary = "docker" + sh """ + ${dockerRun} ${container_type} ${docker_binary} make lint + """ + deleteDir() } +} + +def SphinxDoc() { + node('linux && cpu') { + unstash name: 'srcs' + echo "Running sphinx-doc..." + container_type = "lint" + docker_binary = "docker" + docker_extra_params = "CI_DOCKER_EXTRA_PARAMS_INIT='-e SPHINX_GIT_BRANCH=${BRANCH_NAME}'" + sh """#!/bin/bash + ${docker_extra_params} ${dockerRun} ${container_type} ${docker_binary} make -C doc html + """ + deleteDir() + } +} + +def Doxygen() { + node('linux && cpu') { + unstash name: 'srcs' + echo "Running doxygen..." + container_type = "lint" + docker_binary = "docker" + sh """ + ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/doxygen.sh + """ + deleteDir() + } +} +def BuildCPU() { + node('linux && cpu') { + unstash name: 'srcs' + echo "Build CPU" + container_type = "lint" + docker_binary = "docker" + sh """ + ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/build_via_cmake.sh + ${dockerRun} ${container_type} ${docker_binary} build/testxgboost + """ + deleteDir() + } +} + +def BuildCUDA(args) { + node('linux && cpu') { + unstash name: 'srcs' + echo "Build with CUDA ${args.cuda_version}" + container_type = "gpu_build" + docker_binary = "docker" + docker_args = "--build-arg CUDA_VERSION=${args.cuda_version}" + sh """ + ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_via_cmake.sh -DUSE_CUDA=ON + ${dockerRun} ${container_type} ${docker_binary} ${docker_args} bash -c "cd python-package && rm -rf dist/* && python setup.py bdist_wheel --universal" + """ + // Only stash wheel for CUDA 8.0 target + if (args.cuda_version == '8.0') { + echo 'Stashing Python wheel...' + stash name: 'xgboost_whl', includes: 'python-package/dist/*.whl' + echo 'Stashing C++ test executable (testxgboost)...' + stash name: 'xgboost_cpp_tests', includes: 'bin/testxgboost' + } + deleteDir() + } +} + +def BuildJVMPackages(args) { + node('linux && cpu') { + unstash name: 'srcs' + echo "Build XGBoost4J-Spark with Spark ${args.spark_version}" + container_type = "jvm" + docker_binary = "docker" + // Use only 4 CPU cores + docker_extra_params = "CI_DOCKER_EXTRA_PARAMS_INIT='--cpuset-cpus 0-3'" + sh """ + ${docker_extra_params} ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/build_jvm_packages.sh + """ + echo 'Stashing XGBoost4J JAR...' + stash name: 'xgboost4j_jar', includes: 'jvm-packages/xgboost4j/target/*.jar,jvm-packages/xgboost4j-spark/target/*.jar,jvm-packages/xgboost4j-example/target/*.jar' + deleteDir() + } +} + +def BuildJVMDoc() { + node('linux && cpu') { + unstash name: 'srcs' + echo "Building JVM doc..." + container_type = "jvm" + docker_binary = "docker" + sh """ + ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/build_jvm_doc.sh ${BRANCH_NAME} + """ + archiveArtifacts artifacts: "jvm-packages/${BRANCH_NAME}.tar.bz2", allowEmptyArchive: true + echo 'Uploading doc...' + s3Upload file: "jvm-packages/${BRANCH_NAME}.tar.bz2", bucket: 'xgboost-docs', acl: 'PublicRead', path: "${BRANCH_NAME}.tar.bz2" + deleteDir() + } +} + +def TestPythonCPU() { + node('linux && cpu') { + echo "Test Python CPU" + } +} + +def TestPythonGPU(args) { + node('linux && cpu') { + echo "Test Python GPU: CUDA ${args.cuda_version}" + } +} + +def TestCppGPU(args) { + node('linux && cpu') { + echo "Test C++, CUDA ${args.cuda_version}" + if (args.multi_gpu) { + echo "Using multiple GPUs" + } + } +} + +def CrossTestJVMwithJDK(args) { + node('linux && cpu') { + unstash name: 'xgboost4j_jar' + unstash name: 'srcs' + echo "Test XGBoost4J on a machine with JDK ${args.jdk_version}" + container_type = "jvm_cross" + docker_binary = "docker" + docker_args = "--build-arg JDK_VERSION=${args.jdk_version}" + // Only run integration tests for JDK 8, as Spark doesn't support later JDKs yet + docker_extra_params = (args.jdk_version == '8') ? "CI_DOCKER_EXTRA_PARAMS_INIT='-e RUN_INTEGRATION_TEST=1'" : "" + sh """ + ${docker_extra_params} ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/test_jvm_cross.sh + """ + deleteDir() + } +} + +def TestR(args) { + node('linux && cpu') { + echo "Test R package: R version ${args.r_version}" + } +} diff --git a/Jenkinsfile-restricted b/Jenkinsfile-restricted deleted file mode 100644 index f55997640d9d..000000000000 --- a/Jenkinsfile-restricted +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/groovy -// -*- mode: groovy -*- -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -import groovy.transform.Field - -/* Restricted tasks: tasks generating artifacts, such as binary wheels and - documentation */ - -// Command to run command inside a docker container -def dockerRun = 'tests/ci_build/ci_build.sh' -// Utility functions -@Field -def utils -@Field -def commit_id -@Field -def branch_name - -def buildMatrix = [ - [ "enabled": true, "os" : "linux", "withGpu": true, "withNccl": true, "withOmp": true, "pythonVersion": "2.7", "cudaVersion": "10.0" ], - [ "enabled": true, "os" : "linux", "withGpu": true, "withNccl": true, "withOmp": true, "pythonVersion": "2.7", "cudaVersion": "9.2" ], - [ "enabled": true, "os" : "linux", "withGpu": true, "withNccl": true, "withOmp": true, "pythonVersion": "2.7", "cudaVersion": "8.0" ], - [ "enabled": true, "os" : "linux", "withGpu": true, "withNccl": false, "withOmp": true, "pythonVersion": "2.7", "cudaVersion": "8.0" ], -] - -pipeline { - // Each stage specify its own agent - agent none - - // Setup common job properties - options { - ansiColor('xterm') - timestamps() - timeout(time: 120, unit: 'MINUTES') - buildDiscarder(logRotator(numToKeepStr: '10')) - } - - // Build stages - stages { - stage('Jenkins: Get sources') { - agent { - label 'restricted' - } - steps { - script { - utils = load('tests/ci_build/jenkins_tools.Groovy') - utils.checkoutSrcs() - commit_id = "${GIT_COMMIT}" - branch_name = "${GIT_LOCAL_BRANCH}" - } - stash name: 'srcs', excludes: '.git/' - milestone label: 'Sources ready', ordinal: 1 - } - } - stage('Jenkins: Build doc') { - steps { - script { - retry(1) { - node('linux && cpu && restricted') { - unstash name: 'srcs' - echo 'Building doc...' - dir ('jvm-packages') { - sh "bash ./build_doc.sh ${commit_id}" - archiveArtifacts artifacts: "${commit_id}.tar.bz2", allowEmptyArchive: true - echo 'Deploying doc...' - withAWS(credentials:'xgboost-doc-bucket') { - s3Upload file: "${commit_id}.tar.bz2", bucket: 'xgboost-docs', acl: 'PublicRead', path: "${branch_name}.tar.bz2" - } - } - } - } - } - } - } - - stage('Jenkins: Build artifacts') { - steps { - script { - parallel (buildMatrix.findAll{it['enabled']}.collectEntries{ c -> - def buildName = utils.getBuildName(c) - utils.buildFactory(buildName, c, true, this.&buildPlatformCmake) - }) - } - } - } - } -} - -/** - * Build platform and test it via cmake. - */ -def buildPlatformCmake(buildName, conf, nodeReq, dockerTarget) { - def opts = utils.cmakeOptions(conf) - // Destination dir for artifacts - def distDir = "dist/${buildName}" - def dockerArgs = "" - if(conf["withGpu"]){ - dockerArgs = "--build-arg CUDA_VERSION=" + conf["cudaVersion"] - } - // Build node - this is returned result - retry(1) { - node(nodeReq) { - unstash name: 'srcs' - echo """ - |===== XGBoost CMake build ===== - | dockerTarget: ${dockerTarget} - | cmakeOpts : ${opts} - |========================= - """.stripMargin('|') - // Invoke command inside docker - sh """ - ${dockerRun} ${dockerTarget} ${dockerArgs} tests/ci_build/build_via_cmake.sh ${opts} - ${dockerRun} ${dockerTarget} ${dockerArgs} bash -c "cd python-package; rm -f dist/*; python setup.py bdist_wheel --universal" - rm -rf "${distDir}"; mkdir -p "${distDir}/py" - cp xgboost "${distDir}" - cp -r lib "${distDir}" - cp -r python-package/dist "${distDir}/py" - """ - archiveArtifacts artifacts: "${distDir}/**/*.*", allowEmptyArchive: true - } - } -} diff --git a/Makefile b/Makefile index 6c8477186675..42d3bfe1a0ca 100644 --- a/Makefile +++ b/Makefile @@ -176,7 +176,11 @@ rcpplint: python3 dmlc-core/scripts/lint.py xgboost ${LINT_LANG} R-package/src lint: rcpplint - python3 dmlc-core/scripts/lint.py --pylint-rc ${PWD}/python-package/.pylintrc xgboost ${LINT_LANG} include src plugin python-package + python3 dmlc-core/scripts/lint.py --exclude_path python-package/xgboost/dmlc-core \ + python-package/xgboost/include python-package/xgboost/lib \ + python-package/xgboost/make python-package/xgboost/rabit \ + python-package/xgboost/src --pylint-rc ${PWD}/python-package/.pylintrc xgboost \ + ${LINT_LANG} include src plugin python-package pylint: flake8 --ignore E501 python-package diff --git a/doc/conf.py b/doc/conf.py index 7e0d81a769ea..0ccfe8039e66 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -22,8 +22,11 @@ import shlex import guzzle_sphinx_theme -git_branch = [re.sub(r'origin/', '', x.lstrip(' ')) for x in str(git.branch('-r', '--contains', 'HEAD')).rstrip('\n').split('\n')] -git_branch = [x for x in git_branch if 'HEAD' not in x] +git_branch = os.getenv('SPHINX_GIT_BRANCH', default=None) +if git_branch is None: + # If SPHINX_GIT_BRANCH environment variable is not given, run git to determine branch name + git_branch = [re.sub(r'origin/', '', x.lstrip(' ')) for x in str(git.branch('-r', '--contains', 'HEAD')).rstrip('\n').split('\n')] + git_branch = [x for x in git_branch if 'HEAD' not in x] print('git_branch = {}'.format(git_branch[0])) try: filename, _ = urllib.request.urlretrieve('https://s3-us-west-2.amazonaws.com/xgboost-docs/{}.tar.bz2'.format(git_branch[0])) diff --git a/jvm-packages/xgboost4j-example/src/main/java/ml/dmlc/xgboost4j/java/example/ExternalMemory.java b/jvm-packages/xgboost4j-example/src/main/java/ml/dmlc/xgboost4j/java/example/ExternalMemory.java index a4a1cb703850..349098ae1386 100644 --- a/jvm-packages/xgboost4j-example/src/main/java/ml/dmlc/xgboost4j/java/example/ExternalMemory.java +++ b/jvm-packages/xgboost4j-example/src/main/java/ml/dmlc/xgboost4j/java/example/ExternalMemory.java @@ -32,8 +32,8 @@ public static void main(String[] args) throws XGBoostError { //this is the only difference, add a # followed by a cache prefix name //several cache file with the prefix will be generated //currently only support convert from libsvm file - DMatrix trainMat = new DMatrix("../demo/data/agaricus.txt.train#dtrain.cache"); - DMatrix testMat = new DMatrix("../demo/data/agaricus.txt.test#dtest.cache"); + DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train#dtrain.cache"); + DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test#dtest.cache"); //specify parameters HashMap params = new HashMap(); diff --git a/jvm-packages/xgboost4j-tester/generate_pom.py b/jvm-packages/xgboost4j-tester/generate_pom.py new file mode 100644 index 000000000000..3f1099c61a24 --- /dev/null +++ b/jvm-packages/xgboost4j-tester/generate_pom.py @@ -0,0 +1,208 @@ +import sys + +pom_template = """ + + + + 4.0.0 + + ml.dmlc + xgboost4j-tester + 1.0-SNAPSHOT + + xgboost4j-tester + + + UTF-8 + {maven_compiler_source} + {maven_compiler_target} + {spark_version} + {scala_version} + {scala_binary_version} + + + + + com.esotericsoftware.kryo + kryo + 2.21 + + + org.scala-lang + scala-compiler + ${{scala.version}} + + + org.scala-lang + scala-reflect + ${{scala.version}} + + + org.scala-lang + scala-library + ${{scala.version}} + + + commons-logging + commons-logging + 1.2 + + + com.typesafe.akka + akka-actor_${{scala.binary.version}} + 2.3.11 + compile + + + com.typesafe.akka + akka-testkit_${{scala.binary.version}} + 2.3.11 + test + + + org.scalatest + scalatest_${{scala.binary.version}} + 3.0.0 + test + + + org.apache.commons + commons-lang3 + 3.4 + + + org.apache.spark + spark-core_${{scala.binary.version}} + ${{spark.version}} + provided + + + org.apache.spark + spark-sql_${{scala.binary.version}} + ${{spark.version}} + provided + + + org.apache.spark + spark-mllib_${{scala.binary.version}} + ${{spark.version}} + provided + + + junit + junit + 4.11 + test + + + ml.dmlc + xgboost4j + {xgboost4j_version} + + + ml.dmlc + xgboost4j + {xgboost4j_version} + tests + test-jar + test + + + ml.dmlc + xgboost4j-spark + {xgboost4j_version} + + + ml.dmlc + xgboost4j-example + {xgboost4j_version} + + + + + + + + maven-clean-plugin + 3.1.0 + + + + maven-resources-plugin + 3.0.2 + + + maven-compiler-plugin + 3.8.0 + + + maven-jar-plugin + 3.0.2 + + + maven-install-plugin + 2.5.2 + + + maven-deploy-plugin + 2.8.2 + + + org.apache.maven.plugins + maven-assembly-plugin + 2.4 + + + jar-with-dependencies + + + + ml.dmlc.xgboost4j.tester.App + + + + + + package + + single + + + + + + + maven-site-plugin + 3.7.1 + + + maven-project-info-reports-plugin + 3.0.0 + + + org.apache.maven.plugins + maven-surefire-plugin + 2.22.1 + + + ml.dmlc:xgboost4j + + + + + + +""" + +if __name__ == '__main__': + if len(sys.argv) != 7: + print('Usage: {} [xgboost4j version] [maven compiler source level] [maven compiler target level] [spark version] [scala version] [scala binary version]'.format(sys.argv[0])) + sys.exit(1) + with open('pom.xml', 'w') as f: + print(pom_template.format(xgboost4j_version=sys.argv[1], + maven_compiler_source=sys.argv[2], + maven_compiler_target=sys.argv[3], + spark_version=sys.argv[4], + scala_version=sys.argv[5], + scala_binary_version=sys.argv[6]), file=f) diff --git a/jvm-packages/xgboost4j-tester/get_iris.py b/jvm-packages/xgboost4j-tester/get_iris.py new file mode 100644 index 000000000000..f234bb95e198 --- /dev/null +++ b/jvm-packages/xgboost4j-tester/get_iris.py @@ -0,0 +1,10 @@ +from sklearn.datasets import load_iris +import numpy as np +import pandas + +X, y = load_iris(return_X_y=True) +y = y.astype(np.int) +df = pandas.DataFrame(data=X, columns=['sepal length', 'sepal width', 'petal length', 'petal width']) +class_id_to_name = {0:'Iris-setosa', 1:'Iris-versicolor', 2:'Iris-virginica'} +df['class'] = np.vectorize(class_id_to_name.get)(y) +df.to_csv('./iris.csv', float_format='%.1f', header=False, index=False) diff --git a/jvm-packages/xgboost4j-tester/src/main/java/ml/dmlc/xgboost4j/tester/App.java b/jvm-packages/xgboost4j-tester/src/main/java/ml/dmlc/xgboost4j/tester/App.java new file mode 100644 index 000000000000..917f5062061c --- /dev/null +++ b/jvm-packages/xgboost4j-tester/src/main/java/ml/dmlc/xgboost4j/tester/App.java @@ -0,0 +1,26 @@ +package ml.dmlc.xgboost4j.tester; + +import ml.dmlc.xgboost4j.java.example.*; + +import java.io.IOException; +import ml.dmlc.xgboost4j.java.XGBoostError; + +public class App { + public static void main(String[] args) throws IOException, XGBoostError { + String[] args2 = new String[0]; + System.out.println("BoostFromPrediction"); + BoostFromPrediction.main(args2); + System.out.println("CrossValidation"); + CrossValidation.main(args2); + System.out.println("CustomObjective"); + CustomObjective.main(args2); + System.out.println("ExternalMemory"); + ExternalMemory.main(args2); + System.out.println("GeneralizedLinearModel"); + GeneralizedLinearModel.main(args2); + System.out.println("PredictFirstNtree"); + PredictFirstNtree.main(args2); + System.out.println("PredictLeafIndices"); + PredictLeafIndices.main(args2); + } +} diff --git a/jvm-packages/xgboost4j-tester/src/test/java/ml/dmlc/xgboost4j/tester/AppTest.java b/jvm-packages/xgboost4j-tester/src/test/java/ml/dmlc/xgboost4j/tester/AppTest.java new file mode 100644 index 000000000000..2df69374806a --- /dev/null +++ b/jvm-packages/xgboost4j-tester/src/test/java/ml/dmlc/xgboost4j/tester/AppTest.java @@ -0,0 +1,20 @@ +package ml.dmlc.xgboost4j.tester; + +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +/** + * Unit test for simple App. + */ +public class AppTest +{ + /** + * Rigorous Test :-) + */ + @Test + public void shouldAnswerWithTrue() + { + assertTrue( true ); + } +} diff --git a/jvm-packages/xgboost4j/pom.xml b/jvm-packages/xgboost4j/pom.xml index 83372a88ca7a..2a1f0068282d 100644 --- a/jvm-packages/xgboost4j/pom.xml +++ b/jvm-packages/xgboost4j/pom.xml @@ -71,6 +71,18 @@ + + org.apache.maven.plugins + maven-jar-plugin + 3.0.2 + + + + test-jar + + + + org.apache.maven.plugins maven-resources-plugin diff --git a/tests/ci_build/Dockerfile.clang_tidy b/tests/ci_build/Dockerfile.clang_tidy index ec68e2c666dc..18941999f81f 100644 --- a/tests/ci_build/Dockerfile.clang_tidy +++ b/tests/ci_build/Dockerfile.clang_tidy @@ -16,17 +16,6 @@ RUN \ update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-7 100 && \ update-alternatives --install /usr/bin/clang clang /usr/bin/clang-7 100 -# NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html) -RUN \ - export CUDA_SHORT=`echo $CUDA_VERSION | egrep -o '[0-9]+\.[0-9]'` && \ - if [ "${CUDA_SHORT}" != "10.0" ]; then \ - wget https://developer.download.nvidia.com/compute/redist/nccl/v2.2/nccl_2.2.13-1%2Bcuda${CUDA_SHORT}_x86_64.txz && \ - tar xf "nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64.txz" && \ - cp nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64/include/nccl.h /usr/include && \ - cp nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64/lib/* /usr/lib && \ - rm -f nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64.txz && \ - rm -r nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64; fi - # Install Python packages RUN \ pip3 install pyyaml diff --git a/tests/ci_build/Dockerfile.gpu b/tests/ci_build/Dockerfile.gpu_build similarity index 94% rename from tests/ci_build/Dockerfile.gpu rename to tests/ci_build/Dockerfile.gpu_build index 1bf705558ac0..ffc3442681a5 100644 --- a/tests/ci_build/Dockerfile.gpu +++ b/tests/ci_build/Dockerfile.gpu_build @@ -12,8 +12,8 @@ RUN \ yum -y update && \ yum install -y devtoolset-4-gcc devtoolset-4-binutils devtoolset-4-gcc-c++ && \ # Python - wget https://repo.continuum.io/miniconda/Miniconda2-4.3.27-Linux-x86_64.sh && \ - bash Miniconda2-4.3.27-Linux-x86_64.sh -b -p /opt/python && \ + wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \ + bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python && \ # CMake wget -nv -nc https://cmake.org/files/v3.12/cmake-3.12.0-Linux-x86_64.sh --no-check-certificate && \ bash cmake-3.12.0-Linux-x86_64.sh --skip-license --prefix=/usr diff --git a/tests/ci_build/Dockerfile.jvm b/tests/ci_build/Dockerfile.jvm new file mode 100644 index 000000000000..17a54830ed84 --- /dev/null +++ b/tests/ci_build/Dockerfile.jvm @@ -0,0 +1,43 @@ +FROM centos:6 + +# Install all basic requirements +RUN \ + yum -y update && \ + yum install -y tar unzip wget xz git centos-release-scl yum-utils java-1.8.0-openjdk-devel && \ + yum-config-manager --enable centos-sclo-rh-testing && \ + yum -y update && \ + yum install -y devtoolset-4-gcc devtoolset-4-binutils devtoolset-4-gcc-c++ && \ + # Python + wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \ + bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python && \ + # CMake + wget -nv -nc https://cmake.org/files/v3.12/cmake-3.12.0-Linux-x86_64.sh --no-check-certificate && \ + bash cmake-3.12.0-Linux-x86_64.sh --skip-license --prefix=/usr && \ + # Maven + wget http://apache.osuosl.org/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz && \ + tar xvf apache-maven-3.6.1-bin.tar.gz -C /opt && \ + ln -s /opt/apache-maven-3.6.1/ /opt/maven + +ENV PATH=/opt/python/bin:/opt/maven/bin:$PATH +ENV CC=/opt/rh/devtoolset-4/root/usr/bin/gcc +ENV CXX=/opt/rh/devtoolset-4/root/usr/bin/c++ +ENV CPP=/opt/rh/devtoolset-4/root/usr/bin/cpp + +# Install Python packages +RUN \ + pip install numpy pytest scipy scikit-learn wheel kubernetes urllib3==1.22 awscli + +ENV GOSU_VERSION 1.10 + +# Install lightweight sudo (not bound to TTY) +RUN set -ex; \ + wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \ + chmod +x /usr/local/bin/gosu && \ + gosu nobody true + +# Default entry-point to use if running locally +# It will preserve attributes of created files +COPY entrypoint.sh /scripts/ + +WORKDIR /workspace +ENTRYPOINT ["/scripts/entrypoint.sh"] diff --git a/tests/ci_build/Dockerfile.jvm_cross b/tests/ci_build/Dockerfile.jvm_cross new file mode 100644 index 000000000000..69ca260a3f8e --- /dev/null +++ b/tests/ci_build/Dockerfile.jvm_cross @@ -0,0 +1,45 @@ +FROM ubuntu:18.04 +ARG JDK_VERSION=8 + +# Environment +ENV DEBIAN_FRONTEND noninteractive + +# Install all basic requirements +RUN \ + apt-get update && \ + apt-get install -y tar unzip wget openjdk-$JDK_VERSION-jdk libgomp1 && \ + # Python + wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \ + bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python && \ + # Maven + wget http://apache.osuosl.org/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz && \ + tar xvf apache-maven-3.6.1-bin.tar.gz -C /opt && \ + ln -s /opt/apache-maven-3.6.1/ /opt/maven && \ + # Spark + wget http://apache.cs.utah.edu/spark/spark-2.4.1/spark-2.4.1-bin-hadoop2.7.tgz && \ + tar xvf spark-2.4.1-bin-hadoop2.7.tgz -C /opt && \ + ln -s /opt/spark-2.4.1-bin-hadoop2.7 /opt/spark + +ENV PATH=/opt/python/bin:/opt/spark/bin:/opt/maven/bin:$PATH + +# Install Python packages +RUN \ + pip install numpy scipy pandas scikit-learn + +ENV GOSU_VERSION 1.10 + +# Install lightweight sudo (not bound to TTY) +RUN set -ex; \ + wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \ + chmod +x /usr/local/bin/gosu && \ + gosu nobody true + +# Set default JDK version +RUN update-java-alternatives -v -s java-1.$JDK_VERSION.0-openjdk-amd64 + +# Default entry-point to use if running locally +# It will preserve attributes of created files +COPY entrypoint.sh /scripts/ + +WORKDIR /workspace +ENTRYPOINT ["/scripts/entrypoint.sh"] diff --git a/tests/ci_build/Dockerfile.lint b/tests/ci_build/Dockerfile.lint new file mode 100644 index 000000000000..ab638677f920 --- /dev/null +++ b/tests/ci_build/Dockerfile.lint @@ -0,0 +1,31 @@ +FROM ubuntu:18.04 + +# Environment +ENV DEBIAN_FRONTEND noninteractive + +# Install all basic requirements +RUN \ + apt-get update && \ + apt-get install -y tar unzip wget git build-essential python3 python3-pip doxygen graphviz && \ + wget -nv -nc https://cmake.org/files/v3.12/cmake-3.12.0-Linux-x86_64.sh --no-check-certificate && \ + bash cmake-3.12.0-Linux-x86_64.sh --skip-license --prefix=/usr + +# Install Python packages +RUN \ + pip3 install pyyaml cpplint pylint astroid sphinx numpy scipy sh recommonmark guzzle_sphinx_theme mock breathe matplotlib graphviz && \ + pip3 install https://s3.amazonaws.com/h2o-release/datatable/stable/datatable-0.3.2/datatable-0.3.2-cp36-cp36m-linux_x86_64.whl + +ENV GOSU_VERSION 1.10 + +# Install lightweight sudo (not bound to TTY) +RUN set -ex; \ + wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \ + chmod +x /usr/local/bin/gosu && \ + gosu nobody true + +# Default entry-point to use if running locally +# It will preserve attributes of created files +COPY entrypoint.sh /scripts/ + +WORKDIR /workspace +ENTRYPOINT ["/scripts/entrypoint.sh"] diff --git a/tests/ci_build/Dockerfile.release b/tests/ci_build/Dockerfile.release index baf4d3d94461..f1da067898ca 100644 --- a/tests/ci_build/Dockerfile.release +++ b/tests/ci_build/Dockerfile.release @@ -1,8 +1,5 @@ FROM centos:6 -# Environment -ENV DEBIAN_FRONTEND noninteractive - # Install all basic requirements RUN \ yum -y update && \ diff --git a/jvm-packages/build_doc.sh b/tests/ci_build/build_jvm_doc.sh similarity index 57% rename from jvm-packages/build_doc.sh rename to tests/ci_build/build_jvm_doc.sh index 614ea611424b..a536b0efeeb3 100755 --- a/jvm-packages/build_doc.sh +++ b/tests/ci_build/build_jvm_doc.sh @@ -1,21 +1,27 @@ #!/bin/bash if [ $# -ne 1 ]; then - echo "Usage: $0 [commit id]" + echo "Usage: $0 [branch name]" exit 1 fi set -e set -x -commit_id=$1 +# Initialize local Maven repository +./tests/ci_build/initialize_maven.sh + +rm -rf build/ +cd jvm-packages + +branch_name=$1 # Install JVM packages in local Maven repository -mvn install -DskipTests +mvn --no-transfer-progress install -DskipTests # Build Scaladocs -mvn scala:doc -DskipTests +mvn --no-transfer-progress scala:doc -DskipTests # Build Javadocs -mvn javadoc:javadoc -DskipTests +mvn --no-transfer-progress javadoc:javadoc -DskipTests # Package JVM docs in a tarball mkdir -p tmp/scaladocs @@ -25,8 +31,8 @@ cp -rv xgboost4j-spark/target/site/scaladocs/ ./tmp/scaladocs/xgboost4j-spark/ cp -rv xgboost4j-flink/target/site/scaladocs/ ./tmp/scaladocs/xgboost4j-flink/ cd tmp -tar cvjf ${commit_id}.tar.bz2 javadocs/ scaladocs/ -mv ${commit_id}.tar.bz2 .. +tar cvjf ${branch_name}.tar.bz2 javadocs/ scaladocs/ +mv ${branch_name}.tar.bz2 .. cd .. rm -rfv tmp/ diff --git a/tests/ci_build/build_jvm_packages.sh b/tests/ci_build/build_jvm_packages.sh new file mode 100755 index 000000000000..417d71b663f4 --- /dev/null +++ b/tests/ci_build/build_jvm_packages.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e +set -x + +# Initialize local Maven repository +./tests/ci_build/initialize_maven.sh + +rm -rf build/ +cd jvm-packages + +mvn --no-transfer-progress package -DskipTests + +set +x +set +e diff --git a/tests/ci_build/build_via_cmake.sh b/tests/ci_build/build_via_cmake.sh index 46bdd152cb87..cbfc9725548d 100755 --- a/tests/ci_build/build_via_cmake.sh +++ b/tests/ci_build/build_via_cmake.sh @@ -16,5 +16,5 @@ mkdir build cd build cmake .. "$@" -DGOOGLE_TEST=ON -DGTEST_ROOT=$PWD/../gtest -DCMAKE_VERBOSE_MAKEFILE=ON make clean -make -j +make -j$(nproc) cd .. diff --git a/tests/ci_build/ci_build.sh b/tests/ci_build/ci_build.sh index 676613d9ab8f..14ae47132cec 100755 --- a/tests/ci_build/ci_build.sh +++ b/tests/ci_build/ci_build.sh @@ -2,12 +2,14 @@ # # Execute command within a docker container # -# Usage: ci_build.sh [--dockerfile ] [-it] -# +# Usage: ci_build.sh +# [--dockerfile ] [-it] # # CONTAINER_TYPE: Type of the docker container used the run the build: e.g., # (cpu | gpu) # +# DOCKER_BINARY: Command to invoke docker, e.g. (docker | nvidia-docker). +# # DOCKERFILE_PATH: (Optional) Path to the Dockerfile used for docker build. If # this optional value is not supplied (via the --dockerfile # flag), will use Dockerfile.CONTAINER_TYPE in default @@ -24,6 +26,10 @@ shift 1 DOCKERFILE_PATH="${SCRIPT_DIR}/Dockerfile.${CONTAINER_TYPE}" DOCKER_CONTEXT_PATH="${SCRIPT_DIR}" +# Get docker binary command (should be either docker or nvidia-docker) +DOCKER_BINARY="$1" +shift 1 + if [[ "$1" == "--dockerfile" ]]; then DOCKERFILE_PATH="$2" DOCKER_CONTEXT_PATH=$(dirname "${DOCKERFILE_PATH}") @@ -32,6 +38,11 @@ if [[ "$1" == "--dockerfile" ]]; then shift 2 fi +if [[ -n "${CI_DOCKER_EXTRA_PARAMS_INIT}" ]] +then + IFS=' ' read -r -a CI_DOCKER_EXTRA_PARAMS <<< "${CI_DOCKER_EXTRA_PARAMS_INIT}" +fi + if [[ "$1" == "-it" ]]; then CI_DOCKER_EXTRA_PARAMS+=('-it') shift 1 @@ -61,13 +72,6 @@ if [ "$#" -lt 1 ] || [ ! -e "${SCRIPT_DIR}/Dockerfile.${CONTAINER_TYPE}" ]; then exit 1 fi -# Use nvidia-docker if the container is GPU. -if [[ "${CONTAINER_TYPE}" == *"gpu"* ]]; then - DOCKER_BINARY="nvidia-docker" -else - DOCKER_BINARY="docker" -fi - # Helper function to traverse directories up until given file is found. function upsearch () { test / == "$PWD" && return || \ @@ -84,7 +88,9 @@ DOCKER_IMG_NAME="xgb-ci.${CONTAINER_TYPE}" # Append cuda version if available CUDA_VERSION=$(echo "${CI_DOCKER_BUILD_ARG}" | grep CUDA_VERSION | egrep -o '[0-9]*\.[0-9]*') -DOCKER_IMG_NAME=$DOCKER_IMG_NAME$CUDA_VERSION +# Append jdk version if available +JDK_VERSION=$(echo "${CI_DOCKER_BUILD_ARG}" | grep JDK_VERSION | egrep -o '[0-9]*') +DOCKER_IMG_NAME=$DOCKER_IMG_NAME$CUDA_VERSION$JDK_VERSION # Under Jenkins matrix build, the build tag may contain characters such as # commas (,) and equal signs (=), which are not valid inside docker image names. @@ -98,7 +104,7 @@ UBUNTU_ON_WINDOWS=$([ -e /proc/version ] && grep -l Microsoft /proc/version || e # MSYS, Git Bash, etc. MSYS=$([ -e /proc/version ] && grep -l MINGW /proc/version || echo "") -if [[ -z "$UBUNTU_ON_WINDOWS" ]] && [[ -z "$MSYS" ]]; then +if [[ -z "$UBUNTU_ON_WINDOWS" ]] && [[ -z "$MSYS" ]] && [[ ! "$OSTYPE" == "darwin"* ]]; then USER_IDS="-e CI_BUILD_UID=$( id -u ) -e CI_BUILD_GID=$( id -g ) -e CI_BUILD_USER=$( id -un ) -e CI_BUILD_GROUP=$( id -gn ) -e CI_BUILD_HOME=${WORKSPACE}" fi @@ -181,6 +187,7 @@ echo "Running '${COMMAND[*]}' inside ${DOCKER_IMG_NAME}..." # By default we cleanup - remove the container once it finish running (--rm) # and share the PID namespace (--pid=host) so the process inside does not have # pid 1 and SIGKILL is propagated to the process inside (jenkins can kill it). +set -x ${DOCKER_BINARY} run --rm --pid=host \ -v "${WORKSPACE}":/workspace \ -w /workspace \ diff --git a/tests/ci_build/doxygen.sh b/tests/ci_build/doxygen.sh new file mode 100755 index 000000000000..e96fdd9d863e --- /dev/null +++ b/tests/ci_build/doxygen.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -e +set -x + +rm -rf build +mkdir build +cd build +cmake .. -DBUILD_C_DOC=ON +make -j diff --git a/tests/ci_build/initialize_maven.sh b/tests/ci_build/initialize_maven.sh new file mode 100755 index 000000000000..ef15fdf4f1e5 --- /dev/null +++ b/tests/ci_build/initialize_maven.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -e +set -x + +#if [ -z ${CI_BUILD_USER} ] +#then +# echo 'Must be run inside Jenkins CI' +# exit 1 +#fi +gosu root mkdir -p /cache +gosu root chown ${CI_BUILD_USER}:${CI_BUILD_GROUP} /cache + +# Download cached Maven repository, to speed up build +python3 -m awscli s3 cp s3://xgboost-ci-jenkins-artifacts/maven-repo-cache.tar.bz2 /cache/maven-repo-cache.tar.bz2 || true + +if [[ -f "/cache/maven-repo-cache.tar.bz2" ]] +then + tar xvf /cache/maven-repo-cache.tar.bz2 -C ${HOME} +fi diff --git a/tests/ci_build/jenkins_tools.Groovy b/tests/ci_build/jenkins_tools.Groovy index e7ac6a443f22..1bc2574c6ac0 100644 --- a/tests/ci_build/jenkins_tools.Groovy +++ b/tests/ci_build/jenkins_tools.Groovy @@ -6,20 +6,6 @@ // Command to run command inside a docker container dockerRun = 'tests/ci_build/ci_build.sh' -// initialize source codes -def checkoutSrcs() { - retry(5) { - try { - timeout(time: 2, unit: 'MINUTES') { - checkout scm - sh 'git submodule update --init' - } - } catch (exc) { - deleteDir() - error "Failed to fetch source codes" - } - } -} /** * Creates cmake and make builds diff --git a/tests/ci_build/test_jvm_cross.sh b/tests/ci_build/test_jvm_cross.sh new file mode 100755 index 000000000000..50b7f102f371 --- /dev/null +++ b/tests/ci_build/test_jvm_cross.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e +set -x + +# Initialize local Maven repository +./tests/ci_build/initialize_maven.sh + +# Get version number of XGBoost4J and other auxiliary information +cd jvm-packages +xgboost4j_version=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) +maven_compiler_source=$(mvn help:evaluate -Dexpression=maven.compiler.source -q -DforceStdout) +maven_compiler_target=$(mvn help:evaluate -Dexpression=maven.compiler.target -q -DforceStdout) +spark_version=$(mvn help:evaluate -Dexpression=spark.version -q -DforceStdout) +scala_version=$(mvn help:evaluate -Dexpression=scala.version -q -DforceStdout) +scala_binary_version=$(mvn help:evaluate -Dexpression=scala.binary.version -q -DforceStdout) + +# Install XGBoost4J JAR into local Maven repository +mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j -Dversion=${xgboost4j_version} -Dpackaging=jar +mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j-${xgboost4j_version}-tests.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j -Dversion=${xgboost4j_version} -Dpackaging=test-jar -Dclassifier=tests +mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-spark/target/xgboost4j-spark-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-spark -Dversion=${xgboost4j_version} -Dpackaging=jar +mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-example/target/xgboost4j-example-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-example -Dversion=${xgboost4j_version} -Dpackaging=jar + +cd xgboost4j-tester +# Generate pom.xml for XGBoost4J-tester, a dummy project to run XGBoost4J tests +python3 ./generate_pom.py ${xgboost4j_version} ${maven_compiler_source} ${maven_compiler_target} ${spark_version} ${scala_version} ${scala_binary_version} +# Run unit tests +mvn --no-transfer-progress test + +# Run integration tests with XGBoost4J-Spark +if [ ! -z "$RUN_INTEGRATION_TEST" ] +then + mvn --no-transfer-progress package -DskipTests + python3 get_iris.py + spark-submit --class ml.dmlc.xgboost4j.scala.example.spark.SparkTraining --master 'local[8]' ./target/xgboost4j-tester-1.0-SNAPSHOT-jar-with-dependencies.jar ${PWD}/iris.csv + spark-submit --class ml.dmlc.xgboost4j.scala.example.spark.SparkMLlibPipeline --master 'local[8]' ./target/xgboost4j-tester-1.0-SNAPSHOT-jar-with-dependencies.jar ${PWD}/iris.csv ${PWD}/native_model ${PWD}/pipeline_model +fi + +set +x +set +e