Jenkinsfile

// Jenkinsfile to run pull-request status checks
pipeline {
    parameters {
        string(name: 'PROJECT_NAME', defaultValue: 'aimet', description: 'project name')
        string(name: 'BUILD_LABEL_GPU', defaultValue: 'docker-build-aimet-pr-gpu', description: 'Jenkins node host name or label for GPU variants')
        string(name: 'BUILD_LABEL_CPU', defaultValue: 'docker-build-aimet-pr-cpu', description: 'Jenkins node host name or label for CPU variants')
        string(name: 'BUILD_TIMEOUT', defaultValue: '3', description: 'Build timeout in hours')
        string(name: 'USE_LINARO', defaultValue: '-l', description: 'Use pre-built Linaro images, if available.')
        string(name: 'PREBUILT_DOCKER_IMAGE_URL', defaultValue: '', description: 'Use pre-built docker image from URL, if available.')
    }
    agent { label "${params.BUILD_LABEL_CPU}" }
    options {
        timestamps()
        timeout( time: 3, unit: 'HOURS' )
    }
    environment{
        def workspace   = pwd()
        def REPO_URL    = sh (script: "git config --get remote.origin.url",     returnStdout: true).trim()
        def REPO_NAME   = sh (script: "basename -s .git ${REPO_URL}",           returnStdout: true).trim()

        PROJECT_ROOT    = "${workspace}/${REPO_NAME}"
        PROJECT_NAME    = "${params.PROJECT_NAME}"
        WORKSPACE_ROOT  = "${workspace}"
        XUNIT_XSL_FILE  = "${PROJECT_ROOT}/Jenkins/to-junit.xsl"
    }
    stages {
        stage('Setup') {
            steps {
                echo "*** Running SETUP stage on ${env.NODE_NAME} in workspace ${env.WORKSPACE_ROOT} ***"
                sh 'bash -l -c "rm -rf ${PROJECT_ROOT} | true"'
                sh 'bash -l -c "mkdir ${PROJECT_ROOT}"'
                // moving all files and folders except current and parent into ${REPO_NAME} folder
                sh 'mv `ls -1a | grep -v "^${REPO_NAME}$" | grep -v "^\\.$" | grep -v "^\\.\\.$" ` ${REPO_NAME}'
                script {
                    // load threshold file to be used for unit test and code quality reporting
                    def THRESHOLD_FILE = "${PROJECT_ROOT}/Jenkins/jenkins_threshold_configs.json"
                    if (!fileExists("${THRESHOLD_FILE}")) {
                        error("Error: Threshold file must be specified for project. ${THRESHOLD_FILE} does not exist.")
                    }

                    echo "Loading Thresholds Config  file"
                    THRESHOLD_OBJ = readJSON file: "${PROJECT_ROOT}/Jenkins/jenkins_threshold_configs.json";
                }
                stash name: 'AIMETBuildTree', useDefaultExcludes: false
            }
        }

        stage("Check Commits") {
            agent { label "${params.BUILD_LABEL_CPU}" }

            steps {
                //Set up a TF-CPU docker container to run commit checks script on
                cleanWs()
                unstash 'AIMETBuildTree'
                script{
                    env.AIMET_VARIANT_TF_GPU = "tf-cpu"
                }
                echo "*** Running SETUP stage for ${env.AIMET_VARIANT_TF_CPU} variant on ${env.NODE_NAME} in workspace ${env.WORKSPACE_ROOT} ***"
                echo 'Running commit checks...'
                script{
                    env.REPO_OR_FORK_URL = "${REPO_URL}"
                    using_linaro=env.USE_LINARO
                    if (env.CHANGE_FORK != null) {
                        env.REPO_OR_FORK_URL = "https://github.com/${env.CHANGE_FORK}/aimet"
                    }
                    sh "bash -l -c \"git clone ${env.REPO_OR_FORK_URL} -b ${env.CHANGE_BRANCH} commit_checks_repo\""
                    if (didDockerFileChange("tf-cpu")) {
                            // setting USE LINARO value to EMPTY to rebuild docker image
                            using_linaro=""
                    }
                    if ("${env.QCInternalValidation}" == "true") {
                        withCredentials([usernameColonPassword(credentialsId: 'neoci-pat', variable: 'USERPASS')]) {
                            sh "AIMET_VARIANT=\"tf-cpu\" bash -l -c \"cd ${REPO_NAME} && ${env.DEPENDENCY_DATA_PATH}/commit_check/commit-checks.sh -e AIMET_VARIANT ${using_linaro}\""
                        }
                    } else {
                        sh "AIMET_VARIANT=\"tf-cpu\" bash -l -c \"cd ${REPO_NAME} && ${env.DEPENDENCY_DATA_PATH}/commit_check/commit-checks.sh -e AIMET_VARIANT ${using_linaro}\""
                    }
                }
                sh "bash -l -c \"rm -rf commit_checks_repo\""
            }
        }


        stage('Pipelines start') {

            matrix {
                axes {
                    axis {
                        name 'ML_FMWORK'
                        values 'torch', 'tf', 'onnx', 'tf-torch'
                    }
                    axis {
                        name 'PROC_TYPE'
                        values 'cpu', 'gpu'
                    }
                }
                excludes {
                    exclude {
                        axis {
                            name 'ML_FMWORK'
                            values 'tf-torch'
                        }
                        axis {
                            name 'PROC_TYPE'
                            values 'gpu'
                        }
                    }
                }

                agent { label "docker-build-aimet-pr-${PROC_TYPE}" }

                stages {

                    stage('Start') {

                        steps {
                            script {
                                stage("${ML_FMWORK}-${PROC_TYPE}".toUpperCase()) {
                                    print "${ML_FMWORK}-${PROC_TYPE}"
                                }
                            }
                        }

                    }

                    stage('Setup') {

                        steps {
                            cleanWs()
                            unstash 'AIMETBuildTree'
                            echo "*** Running SETUP stage for ${ML_FMWORK}-${PROC_TYPE} variant on ${env.NODE_NAME} in workspace ${env.WORKSPACE_ROOT} ***"
                        }
                    }


                    stage('Build') {

                        steps {
                            echo 'Building code (and generating Docs and pip packages)...'
                            script {
                                runStage("${ML_FMWORK}-${PROC_TYPE}", "-bp")
                            }
                        }
                    }

                    stage('Code violations') {
                        // Works with newer jenkins instances that support the warnings-ng plugin (https://plugins.jenkins.io/warnings-ng)
                        when {
                            expression {
                                env.QCInternalValidation == "false"
                            }
                        }
                        steps {
                            echo 'Running code violations...'
                            script {
                                runStage("${ML_FMWORK}-${PROC_TYPE}", "-v")
                            }
                        }
                        // TODO: Following code needs to be updated to conform to this plugin: https://plugins.jenkins.io/warnings-ng
                        // post {
                        //     always {
                        //         step([
                        //                 $class                      : 'WarningsNgPublisher',
                        //                 parserConfigurations        : [[
                        //                                                 parserName: 'PYLint',
                        //                                                 pattern   : "**/**/**/*pylint_results.out"
                        //                                                 ]],
                        //                 failedTotalHigh             : THRESHOLD_OBJ.pylint_fail_thresholds.high_priority,
                        //                 failedTotalNormal           : THRESHOLD_OBJ.pylint_fail_thresholds.normal_priority,
                        //                 failedTotalLow              : THRESHOLD_OBJ.pylint_fail_thresholds.low_priority,
                        //                 usePreviousBuildAsReference : true
                        //         ])
                        //         script {
                        //             if (currentBuild.currentResult.equals("FAILURE")) {
                        //                 // the plugin won't fail the stage.  it only sets the build status, so we have to fail it
                        //                 // manually
                        //                 sh "exit 1"
                        //             }
                        //         }
                        //     }
                        // }                    
                    }

                    stage('Code violations Legacy') {
                        // Works with older jenkins instances that support the warnings plugin (https://plugins.jenkins.io/warnings)
                        when {
                            expression {
                                env.QCInternalValidation == "true"
                            }
                        }
                        steps {
                            echo 'Running code violations...'
                            script {
                                runStage("${ML_FMWORK}-${PROC_TYPE}", "-v")
                            }
                        }
                        post {
                            always {
                                // NOTE: Works only with https://plugins.jenkins.io/warnings/ (deprecated)
                                step([
                                    $class                      : 'WarningsPublisher',
                                    parserConfigurations        : [[
                                                                    parserName: 'PYLint',
                                                                    pattern: "**/**/**/*pylint_results.out"
                                                                    ]],
                                    failedTotalHigh             : THRESHOLD_OBJ.pylint_fail_thresholds.high_priority,
                                    failedTotalNormal           : THRESHOLD_OBJ.pylint_fail_thresholds.normal_priority,
                                    failedTotalLow              : THRESHOLD_OBJ.pylint_fail_thresholds.low_priority,
                                    usePreviousBuildAsReference : true
                                ])
                                script {
                                    if (currentBuild.currentResult.equals("FAILURE")) {
                                        // the plugin won't fail the stage.  it only sets the build status, so we have to fail it
                                        // manually
                                        sh "exit 1"
                                    }
                                }
                            }
                        }
                    }

                    stage('Unit tests') {

                        steps {
                            script {
                                if ("${env.QCInternalValidation}" == "false") {
                                    echo 'Building code and running tests...'
                                    runStage("${ML_FMWORK}-${PROC_TYPE}", "-u")
                                }
                            }
                        }
                        post {
                            always {
                                script {
                                    if ("${env.QCInternalValidation}" == "false") {
                                        // Parse CTest, Junit, or GoogleTest results
                                        xunit thresholdMode: 2,
                                        thresholds: [
                                            failed(failureNewThreshold: THRESHOLD_OBJ.unittests_thresholds.failed_thresholds.new_tests,
                                                failureThreshold: THRESHOLD_OBJ.unittests_thresholds.failed_thresholds.total_tests,
                                                unstableNewThreshold: THRESHOLD_OBJ.unittests_thresholds.skipped_thresholds.new_tests,
                                                unstableThreshold: THRESHOLD_OBJ.unittests_thresholds.skipped_thresholds.total_tests)],
                                        tools: [Custom(customXSL: "${XUNIT_XSL_FILE}",
                                                    deleteOutputFiles: false,
                                                    excludesPattern: '',
                                                    pattern: "**/unit_test_results/**/*.xml",
                                                    skipNoTestFiles: true,
                                                    stopProcessingIfError: true)]
                                    }
                                }
                            }
                        }
                    }

                    stage('Static analysis') {

                        steps {
                            echo 'Running static analysis...'
                            script {
                                echo 'WARNING: Currently NOT failing on static analysis errors!'
                                //TODO Remove the '| true' from the below invocation when errors are fixed
                                runStage("${ML_FMWORK}-${PROC_TYPE}", "-s | true")
                            }
                        }
                    }

                }
            }
        }


        stage("AIMET extra ALL STAGES") {


            steps {
                script {
                    if ("${env.QCInternalValidation}" == "true") {
                        echo 'Running AIMET additional stages'
                        callAimetExtra(env.CHANGE_TARGET)
                    }
                }
            }

        }
    }
    post {
        always {
            script {
                if (currentBuild.currentResult.equals("SUCCESS")) {
                    echo 'Finished pull request Jenkins file'
                }
            }
        }
    }
}

def didDockerFileChange(aimet_variant) {
    def changedFiles = pullRequest.files.collect {
        it.getFilename()
    }
    echo "Changed files - ${changedFiles}"
    if (changedFiles.contains("Jenkins/Dockerfile.${aimet_variant}".toString())) {
        echo "*** Jenkins/Dockerfile.${aimet_variant} changed in PR, so building docker image locally. ***"
        return true
    }
    echo "*** Jenkins/Dockerfile.${aimet_variant} NOT changed in PR, so using pre-built docker image. ***"
    return false
}

def runStage(aimet_variant, options) {

    echo "*** Running stage ${options} for ${aimet_variant} variant on ${env.NODE_NAME} in workspace ${env.WORKSPACE_ROOT} ***"
    using_linaro=env.USE_LINARO
    if (aimet_variant == "default") {
        sh """
            bash -l -c "cd ${REPO_NAME} && ./buildntest.sh ${options}"
        """

    }
    else {
        if (didDockerFileChange(aimet_variant)) {
            // setting USE LINARO value to EMPTY to rebuild docker image
            using_linaro=""
        }
        sh """
            AIMET_VARIANT=${aimet_variant} bash -l -c "cd ${REPO_NAME} && ./buildntest.sh -e AIMET_VARIANT ${options} ${using_linaro}"
        """
    }
}

def callAimetExtra(target_branch) {
    echo target_branch
    using_linaro=env.USE_LINARO
    if (didDockerFileChange("tf-gpu") || didDockerFileChange("tf-cpu") || didDockerFileChange("torch-gpu") || didDockerFileChange("torch-cpu") || didDockerFileChange("onnx-cpu") || didDockerFileChange("onnx-gpu") ) {
            // setting USE LINARO value to EMPTY to rebuild docker image
            using_linaro=""
        }

    if (target_branch.startsWith("release-aimet")) {
        echo "Running AIMET additional stages on ${CHANGE_TARGET} branch ..."
        build job: "AIMET-Extra", parameters: [string(name: 'AIMET_GIT_COMMIT', value: "${CHANGE_BRANCH}"), string(name: 'PROJECT_BRANCH', value: target_branch), string(name: 'USE_LINARO', value: "${using_linaro}"), string(name: 'PREBUILT_DOCKER_IMAGE_URL', value: "${params.PREBUILT_DOCKER_IMAGE_URL}"), string(name: 'AIMETPRO_BRANCH', value: target_branch)]
    }
    else if (target_branch != "develop") {
        echo "Running AIMET additional stages on ${CHANGE_TARGET} branch ..."
        build job: "AIMET-Extra", parameters: [string(name: 'AIMET_GIT_COMMIT', value: "${CHANGE_BRANCH}"), string(name: 'PROJECT_BRANCH', value: target_branch), string(name: 'USE_LINARO', value: "${using_linaro}"), string(name: 'PREBUILT_DOCKER_IMAGE_URL', value: "${params.PREBUILT_DOCKER_IMAGE_URL}")]
    }
    else {
        echo "Running AIMET additional stages on develop branch ..."
        build job: "AIMET-Extra", parameters: [string(name: 'AIMET_GIT_COMMIT', value: "${CHANGE_BRANCH}"), string(name: 'USE_LINARO', value: "${using_linaro}"), string(name: 'PREBUILT_DOCKER_IMAGE_URL', value: "${params.PREBUILT_DOCKER_IMAGE_URL}")]
    }
}