distribution-build.jenkinsfile

/*
 * Copyright OpenSearch Contributors
 * SPDX-License-Identifier: Apache-2.0
 *
 * The OpenSearch Contributors require contributions made to
 * this file be licensed under the Apache-2.0 license or a
 * compatible open source license.
 */

lib = library(identifier: 'jenkins@7.2.0', retriever: modernSCM([
    $class: 'GitSCMSource',
    remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
]))

pipeline {
    options {
        timeout(time: 4, unit: 'HOURS')
    }
    agent none
    environment {
        AGENT_LINUX_X64 = 'Jenkins-Agent-AL2023-X64-M54xlarge-Docker-Host'
        AGENT_LINUX_ARM64 = 'Jenkins-Agent-AL2023-Arm64-M6g4xlarge-Docker-Host'
        AGENT_WINDOWS_X64 = 'Jenkins-Agent-Windows2019-X64-M54xlarge-Docker-Host'
        IMAGE_LINUX_RPM = 'opensearchstaging/ci-runner:ci-runner-almalinux8-opensearch-build-v1' // required for rpm to create digest sha256 correctly with rpm 4.12+, still define here for 1.x version as 2.x+ moved to rockylinux8, later almalinux8, already
        IMAGE_LINUX_DEB = 'opensearchstaging/ci-runner:ci-runner-ubuntu2004-opensearch-build-v2' // required for deb to create pkg using debmake/debuild/debhelper
        IMAGE_WINDOWS_ZIP = 'opensearchstaging/ci-runner:ci-runner-windows2019-opensearch-build-v1' // required for windows to build zip distribution
    }
    parameters {
        string(
            name: 'COMPONENT_NAME',
            description: 'If this field contains one or more component names (e.g. OpenSearch common-utils ...), will build with "--component <COMPONENT_NAME> ...", else build everything in the INPUT_MANIFEST.',
            trim: true
        )
        string(
            name: 'INPUT_MANIFEST',
            description: 'Input manifest under the manifests folder, e.g. 2.0.0/opensearch-2.0.0.yml.',
            trim: true
        )
        string(
            name: 'TEST_MANIFEST',
            description: 'Test manifest under the manifests folder, e.g. 2.0.0/opensearch-2.0.0-test.yml.',
            trim: true
        )
        string(
            name: 'INTEG_TEST_JOB_NAME',
            description: 'Name of integration test job that will be triggered, e.g. Playground/integ-test. A non-null empty value here will skip integration tests.',
            defaultValue: 'integ-test',
            trim: true
        )
        string(
            name: 'BWC_TEST_JOB_NAME',
            description: 'Name of backwards compatibility test job that will be triggered, e.g. Playground/bwc-test.  A non-null empty value here will skip BWC tests.',
            defaultValue: 'bwc-test',
            trim: true
        )
        string( // Note: need to update 'verify-parameters' entries if you add new platform(s)
            name: 'BUILD_PLATFORM',
            description: "Build selected platform, choices include 'linux', 'windows'. Can combine multiple platforms with space in between (docker is only available on linux)",
            defaultValue: 'linux windows',
            trim: true
        )
        string( // Note: need to update 'verify-parameters' entries if you add new distribution(s)
            name: 'BUILD_DISTRIBUTION',
            description: "Build selected distribution, choices include 'tar', 'rpm', 'deb', 'zip'. Can combine multiple distributions with space in between (docker is only available on tar)",
            defaultValue: 'tar rpm deb zip',
            trim: true
        )
        string( // Note: need to update 'verify-parameters' entries if you add new platform(s)
            name: 'TEST_PLATFORM',
            description: "Test selected platform, choices include 'linux', 'windows'. Can combine multiple platforms with space in between (docker is only available on linux)",
            trim: true
        )
        string( // Note: need to update 'verify-parameters' entries if you add new distribution(s)
            name: 'TEST_DISTRIBUTION',
            description: "Build selected distribution, choices include 'tar', 'rpm', 'deb', 'zip'. Can combine multiple distributions with space in between (docker is only available on tar)",
            trim: true
        )
        string(
            name: 'RC_NUMBER',
            description: 'The RC build count. Default is 0 which means its not an RC build.',
            defaultValue: '0'
        )
        choice(
            name: 'BUILD_DOCKER',
            description: 'Build docker image or not with options.',
            choices: ['build_docker', 'build_docker_with_build_number_tag', 'do_not_build_docker'],
        )
        booleanParam(
            name: 'UPDATE_LATEST_URL',
            description: 'Update latest url so /latest/ is pointed to this build',
            defaultValue: true
        )
        booleanParam(
            name: 'UPDATE_GITHUB_ISSUE',
            description: 'To create/close/update a github issue for all component or not.',
            defaultValue: true
        )
        booleanParam(
            name: 'CONTINUE_ON_ERROR',
            description: 'Continue building the distribution even if a one or more component fails',
            defaultValue: true
        )
        booleanParam(
            name: 'INCREMENTAL',
            description: 'Whether to trigger incremental build. Defaults to false.',
            defaultValue: true
        )
        string(
            name: 'PREVIOUS_BUILD_ID',
            description: 'The build ID used to download previous build artifacts. Defaults to latest.',
            defaultValue: 'latest',
            trim: true
        )
    }
    stages {
        stage('verify-parameters') {
            agent {
                docker {
                    label AGENT_LINUX_X64
                    image 'docker/library/alpine:3'
                    registryUrl 'https://public.ecr.aws/'
                    alwaysPull true
                }
            }
            steps {
                script {
                    echo('Detect Docker Images and Related Parameters')
                    dockerAgent = detectDockerAgent()
                    env.javaVersionNumber = dockerAgent.javaVersion.replaceAll('[^0-9]', '') // Only get number
                    currentBuild.description = INPUT_MANIFEST

                    paramType = [
                        'BUILD_PLATFORM': 'linux windows',
                        'BUILD_DISTRIBUTION': 'tar rpm deb zip',
                        'TEST_PLATFORM': 'linux windows',
                        'TEST_DISTRIBUTION': 'tar rpm deb zip',
                    ]

                    // BUILD params are required while TEST params are not
                    if (params.TEST_PLATFORM == null || params.TEST_PLATFORM == '') {
                        paramType.remove('TEST_PLATFORM')
                    }

                    if (params.TEST_DISTRIBUTION == null || params.TEST_DISTRIBUTION == '') {
                        paramType.remove('TEST_DISTRIBUTION')
                    }

                    paramType.each { key, value ->
                        verifyParameterPlatformDistribution(key, value)
                    }
                }
            }
        }
        stage('build') {
            parallel {
                stage('Trigger-min-snapshot-build') {
                    agent {
                        docker {
                            label 'Jenkins-Agent-AL2023-X64-M54xlarge-Docker-Host'
                            image 'docker/library/alpine:3'
                            registryUrl 'https://public.ecr.aws/'
                            alwaysPull true
                        }
                    }
                    steps {
                        script {
                            def snapshotBuild =
                                build job: 'publish-opensearch-min-snapshots',
                                propagate: false,
                                wait: false,
                                parameters: [
                                    string(name: 'INPUT_MANIFEST', value: "${INPUT_MANIFEST}"),
                                ]
                        }
                    }
                }
                stage('build-and-test-linux-x64-tar') {
                    when {
                        beforeAgent true
                        expression {
                            params.BUILD_PLATFORM.contains('linux')
                        }
                        expression {
                            params.BUILD_DISTRIBUTION.contains('tar')
                        }
                    }
                    agent {
                        docker {
                            label AGENT_LINUX_X64
                            image dockerAgent.image
                            args dockerAgent.args
                            registryUrl 'https://public.ecr.aws/'
                            alwaysPull true
                        }
                    }
                    steps {
                        script {
                            def buildManifestObj = buildAssembleUpload(
                                componentName: "${COMPONENT_NAME}",
                                inputManifest: "manifests/${INPUT_MANIFEST}",
                                platform: 'linux',
                                architecture: 'x64',
                                distribution: 'tar',
                                continueOnError: params.CONTINUE_ON_ERROR,
                                incremental: params.INCREMENTAL,
                                previousBuildId: params.PREVIOUS_BUILD_ID
                            )
                            String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER)
                            String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER)
                            env.ARTIFACT_URL_LINUX_X64_TAR = artifactUrl
                            env.INDEX_FILE_PATH_X64_TAR = buildManifestObj.getIndexFileRoot("${JOB_NAME}")

                            echo "buildManifestUrl (linux, x64, tar): ${buildManifestUrl}"
                            echo "artifactUrl (linux, x64, tar): ${artifactUrl}"

                            parallel([
                                'integ-test': {
                                    triggerIntegrationTests(buildManifestUrl, 'linux', 'tar')
                                },
                                'bwc-test': {
                                    triggerBWCTests(buildManifestUrl, 'linux', 'tar')
                                }
                            ])
                        }
                    }
                    post {
                        success {
                            script {
                                if (params.UPDATE_LATEST_URL) {
                                    uploadIndexFile(
                                        indexFilePath: env.INDEX_FILE_PATH_X64_TAR
                                    )
                                }
                            }
                        }
                        unstable {
                            script {
                                if (params.UPDATE_LATEST_URL) {
                                    uploadIndexFile(
                                        indexFilePath: env.INDEX_FILE_PATH_X64_TAR
                                    )
                                }
                            }
                        }
                        always {
                            script {
                                if (params.CONTINUE_ON_ERROR) {
                                    markStageUnstableIfPluginsFailedToBuild()
                                }
                                postCleanup()
                            }
                        }
                    }
                }
                stage('build-and-test-linux-x64-rpm') {
                    when {
                        beforeAgent true
                        expression {
                            params.BUILD_PLATFORM.contains('linux')
                        }
                        expression {
                            params.BUILD_DISTRIBUTION.contains('rpm')
                        }
                    }
                    agent { label AGENT_LINUX_X64 }
                    stages {
                        stage('build-archive-linux-x64-rpm') {
                            agent {
                                docker {
                                    label AGENT_LINUX_X64
                                    image dockerAgent.image
                                    args dockerAgent.args
                                    registryUrl 'https://public.ecr.aws/'
                                    alwaysPull true
                                }
                            }
                            steps {
                                script {
                                    buildArchive(
                                        componentName: "${COMPONENT_NAME}",
                                        inputManifest: "manifests/${INPUT_MANIFEST}",
                                        platform: 'linux',
                                        architecture: 'x64',
                                        distribution: 'rpm',
                                        continueOnError: params.CONTINUE_ON_ERROR,
                                        incremental: params.INCREMENTAL,
                                        previousBuildId: params.PREVIOUS_BUILD_ID,
                                        stashName: "build-archive-linux-x64-rpm-${JOB_NAME}-${BUILD_NUMBER}"
                                    )
                                }
                            }
                            post {
                                always {
                                    script {
                                        if (params.CONTINUE_ON_ERROR) {
                                            markStageUnstableIfPluginsFailedToBuild()
                                        }
                                        postCleanup()
                                    }
                                }
                            }
                        }
                        stage('assemble-archive-and-test-linux-x64-rpm') {
                            agent {
                                docker {
                                    label AGENT_LINUX_X64
                                    image IMAGE_LINUX_RPM
                                    registryUrl 'https://public.ecr.aws/'
                                    alwaysPull true
                                }
                            }
                            steps {
                                script {
                                    def buildManifestObj = archiveAssembleUpload(
                                        componentName: "${COMPONENT_NAME}",
                                        inputManifest: "manifests/${INPUT_MANIFEST}",
                                        platform: 'linux',
                                        architecture: 'x64',
                                        distribution: 'rpm',
                                        continueOnError: params.CONTINUE_ON_ERROR,
                                        stashName: "build-archive-linux-x64-rpm-${JOB_NAME}-${BUILD_NUMBER}"
                                    )
                                    String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER)
                                    String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER)
                                    env.ARTIFACT_URL_LINUX_X64_RPM = artifactUrl
                                    env.INDEX_FILE_PATH_X64_RPM = buildManifestObj.getIndexFileRoot("${JOB_NAME}")

                                    echo "buildManifestUrl (linux, x64, rpm): ${buildManifestUrl}"
                                    echo "artifactUrl (linux, x64, rpm): ${artifactUrl}"

                                    String bundleManifestUrl = buildManifestObj.getBundleManifestUrl(JOB_NAME, BUILD_NUMBER)

                                    triggerIntegrationTests(buildManifestUrl, 'linux', 'rpm')
                                }
                            }
                            post {
                                success {
                                    script {
                                        if (params.UPDATE_LATEST_URL) {
                                            uploadIndexFile(
                                                indexFilePath: env.INDEX_FILE_PATH_X64_RPM
                                            )
                                        }
                                    }
                                }
                                unstable {
                                    script {
                                        if (params.UPDATE_LATEST_URL) {
                                            uploadIndexFile(
                                                indexFilePath: env.INDEX_FILE_PATH_X64_RPM
                                            )
                                        }
                                    }
                                }
                                always {
                                    script {
                                        postCleanup()
                                    }
                                }
                            }
                        }
                    }
                }
                stage('build-and-test-linux-x64-deb') {
                    when {
                        beforeAgent true
                        expression {
                            params.BUILD_PLATFORM.contains('linux')
                        }
                        expression {
                            params.BUILD_DISTRIBUTION.contains('deb')
                        }
                    }
                    agent { label AGENT_LINUX_X64 }
                    stages {
                        stage('build-archive-linux-x64-deb') {
                            agent {
                                docker {
                                    label AGENT_LINUX_X64
                                    image dockerAgent.image
                                    args dockerAgent.args
                                    registryUrl 'https://public.ecr.aws/'
                                    alwaysPull true
                                }
                            }
                            steps {
                                script {
                                    buildArchive(
                                        componentName: "${COMPONENT_NAME}",
                                        inputManifest: "manifests/${INPUT_MANIFEST}",
                                        platform: 'linux',
                                        architecture: 'x64',
                                        distribution: 'deb',
                                        continueOnError: params.CONTINUE_ON_ERROR,
                                        incremental: params.INCREMENTAL,
                                        previousBuildId: params.PREVIOUS_BUILD_ID,
                                        stashName: "build-archive-linux-x64-deb-${JOB_NAME}-${BUILD_NUMBER}"
                                    )
                                }
                            }
                            post {
                                always {
                                    script {
                                        if (params.CONTINUE_ON_ERROR) {
                                            markStageUnstableIfPluginsFailedToBuild()
                                        }
                                        postCleanup()
                                    }
                                }
                            }
                        }
                        stage('assemble-archive-and-test-linux-x64-deb') {
                            agent {
                                docker {
                                    label AGENT_LINUX_X64
                                    image IMAGE_LINUX_DEB
                                    registryUrl 'https://public.ecr.aws/'
                                    alwaysPull true
                                }
                            }
                            steps {
                                script {
                                    def buildManifestObj = archiveAssembleUpload(
                                        componentName: "${COMPONENT_NAME}",
                                        inputManifest: "manifests/${INPUT_MANIFEST}",
                                        platform: 'linux',
                                        architecture: 'x64',
                                        distribution: 'deb',
                                        continueOnError: params.CONTINUE_ON_ERROR,
                                        stashName: "build-archive-linux-x64-deb-${JOB_NAME}-${BUILD_NUMBER}"
                                    )
                                    String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER)
                                    String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER)
                                    env.ARTIFACT_URL_LINUX_X64_DEB = artifactUrl
                                    env.INDEX_FILE_PATH_X64_DEB = buildManifestObj.getIndexFileRoot("${JOB_NAME}")

                                    echo "buildManifestUrl (linux, x64, deb): ${buildManifestUrl}"
                                    echo "artifactUrl (linux, x64, deb): ${artifactUrl}"

                                    String bundleManifestUrl = buildManifestObj.getBundleManifestUrl(JOB_NAME, BUILD_NUMBER)

                                    triggerIntegrationTests(buildManifestUrl, 'linux', 'deb')
                                }
                            }
                            post {
                                success {
                                    script {
                                        if (params.UPDATE_LATEST_URL) {
                                            uploadIndexFile(
                                                indexFilePath: env.INDEX_FILE_PATH_X64_DEB
                                            )
                                        }
                                    }
                                }
                                unstable {
                                    script {
                                        if (params.UPDATE_LATEST_URL) {
                                            uploadIndexFile(
                                                indexFilePath: env.INDEX_FILE_PATH_X64_DEB
                                            )
                                        }
                                    }
                                }
                                always {
                                    script {
                                        postCleanup()
                                    }
                                }
                            }
                        }
                    }
                }
                stage('build-and-test-linux-arm64-tar') {
                    when {
                        beforeAgent true
                        expression {
                            params.BUILD_PLATFORM.contains('linux')
                        }
                        expression {
                            params.BUILD_DISTRIBUTION.contains('tar')
                        }
                    }
                    agent {
                        docker {
                            label AGENT_LINUX_ARM64
                            image dockerAgent.image
                            args dockerAgent.args
                            registryUrl 'https://public.ecr.aws/'
                            alwaysPull true
                        }
                    }
                    steps {
                        script {
                            def buildManifestObj = buildAssembleUpload(
                                componentName: "${COMPONENT_NAME}",
                                inputManifest: "manifests/${INPUT_MANIFEST}",
                                platform: 'linux',
                                architecture: 'arm64',
                                distribution: 'tar',
                                continueOnError: params.CONTINUE_ON_ERROR,
                                incremental: params.INCREMENTAL,
                                previousBuildId: params.PREVIOUS_BUILD_ID
                            )
                            String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER)
                            String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER)
                            env.ARTIFACT_URL_LINUX_ARM64_TAR = artifactUrl
                            env.INDEX_FILE_PATH_ARM64_TAR = buildManifestObj.getIndexFileRoot("${JOB_NAME}")

                            echo "buildManifestUrl (linux, arm64, tar): ${buildManifestUrl}"
                            echo "artifactUrl (linux, arm64, tar): ${artifactUrl}"

                            triggerIntegrationTests(buildManifestUrl, 'linux', 'tar')
                        }
                    }
                    post {
                        success {
                            script {
                                if (params.UPDATE_LATEST_URL) {
                                    uploadIndexFile(
                                        indexFilePath: env.INDEX_FILE_PATH_ARM64_TAR
                                    )
                                }
                            }
                        }
                        unstable {
                            script {
                                if (params.UPDATE_LATEST_URL) {
                                    uploadIndexFile(
                                        indexFilePath: env.INDEX_FILE_PATH_ARM64_TAR
                                    )
                                }
                            }
                        }
                        always {
                            script {
                                if (params.CONTINUE_ON_ERROR) {
                                    markStageUnstableIfPluginsFailedToBuild()
                                }
                                postCleanup()
                            }
                        }
                    }
                }
                stage('build-and-test-linux-arm64-rpm') {
                    when {
                        beforeAgent true
                        expression {
                            params.BUILD_PLATFORM.contains('linux')
                        }
                        expression {
                            params.BUILD_DISTRIBUTION.contains('rpm')
                        }
                    }
                    agent { label AGENT_LINUX_X64 }
                    stages {
                        stage('build-archive-linux-arm64-rpm') {
                            agent {
                                docker {
                                    label AGENT_LINUX_ARM64
                                    image dockerAgent.image
                                    args dockerAgent.args
                                    registryUrl 'https://public.ecr.aws/'
                                    alwaysPull true
                                }
                            }
                            steps {
                                script {
                                    buildArchive(
                                        componentName: "${COMPONENT_NAME}",
                                        inputManifest: "manifests/${INPUT_MANIFEST}",
                                        platform: 'linux',
                                        architecture: 'arm64',
                                        distribution: 'rpm',
                                        continueOnError: params.CONTINUE_ON_ERROR,
                                        incremental: params.INCREMENTAL,
                                        previousBuildId: params.PREVIOUS_BUILD_ID,
                                        stashName: "build-archive-linux-arm64-rpm-${JOB_NAME}-${BUILD_NUMBER}"
                                    )
                                }
                            }
                            post {
                                always {
                                    script {
                                        if (params.CONTINUE_ON_ERROR) {
                                            markStageUnstableIfPluginsFailedToBuild()
                                        }
                                        postCleanup()
                                    }
                                }
                            }
                        }
                        stage('assemble-archive-and-test-linux-arm64-rpm') {
                            agent {
                                docker {
                                    label AGENT_LINUX_ARM64
                                    image IMAGE_LINUX_RPM
                                    registryUrl 'https://public.ecr.aws/'
                                    alwaysPull true
                                }
                            }
                            steps {
                                script {
                                    def buildManifestObj = archiveAssembleUpload(
                                        componentName: "${COMPONENT_NAME}",
                                        inputManifest: "manifests/${INPUT_MANIFEST}",
                                        platform: 'linux',
                                        architecture: 'arm64',
                                        distribution: 'rpm',
                                        continueOnError: params.CONTINUE_ON_ERROR,
                                        stashName: "build-archive-linux-arm64-rpm-${JOB_NAME}-${BUILD_NUMBER}"
                                    )
                                    String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER)
                                    String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER)
                                    env.ARTIFACT_URL_LINUX_ARM64_RPM = artifactUrl
                                    env.INDEX_FILE_PATH_ARM64_RPM = buildManifestObj.getIndexFileRoot("${JOB_NAME}")

                                    echo "buildManifestUrl (linux, arm64, rpm): ${buildManifestUrl}"
                                    echo "artifactUrl (linux, arm64, rpm): ${artifactUrl}"

                                    String bundleManifestUrl = buildManifestObj.getBundleManifestUrl(JOB_NAME, BUILD_NUMBER)

                                    triggerIntegrationTests(buildManifestUrl, 'linux', 'rpm')
                                }
                            }
                            post {
                                success {
                                    script {
                                        if (params.UPDATE_LATEST_URL) {
                                            uploadIndexFile(
                                                indexFilePath: env.INDEX_FILE_PATH_ARM64_RPM
                                            )
                                        }
                                    }
                                }
                                unstable {
                                    script {
                                        if (params.UPDATE_LATEST_URL) {
                                            uploadIndexFile(
                                                indexFilePath: env.INDEX_FILE_PATH_ARM64_RPM
                                            )
                                        }
                                    }
                                }
                                always {
                                    script {
                                        postCleanup()
                                    }
                                }
                            }
                        }
                    }
                }
                stage('build-and-test-linux-arm64-deb') {
                    when {
                        beforeAgent true
                        expression {
                            params.BUILD_PLATFORM.contains('linux')
                        }
                        expression {
                            params.BUILD_DISTRIBUTION.contains('deb')
                        }
                        
                    }
                    agent { label AGENT_LINUX_ARM64 }
                    stages {
                        stage('build-archive-linux-arm64-deb') {
                            agent {
                                docker {
                                    label AGENT_LINUX_ARM64
                                    image dockerAgent.image
                                    args dockerAgent.args
                                    registryUrl 'https://public.ecr.aws/'
                                    alwaysPull true
                                }
                            }
                            steps {
                                script {
                                    buildArchive(
                                        componentName: "${COMPONENT_NAME}",
                                        inputManifest: "manifests/${INPUT_MANIFEST}",
                                        platform: 'linux',
                                        architecture: 'arm64',
                                        distribution: 'deb',
                                        continueOnError: params.CONTINUE_ON_ERROR,
                                        incremental: params.INCREMENTAL,
                                        previousBuildId: params.PREVIOUS_BUILD_ID,
                                        stashName: "build-archive-linux-arm64-deb-${JOB_NAME}-${BUILD_NUMBER}"
                                    )
                                }
                            }
                            post {
                                always {
                                    script {
                                        if (params.CONTINUE_ON_ERROR) {
                                            markStageUnstableIfPluginsFailedToBuild()
                                        }
                                        postCleanup()
                                    }
                                }
                            }
                        }
                        stage('assemble-archive-and-test-linux-arm64-deb') {
                            agent {
                                docker {
                                    label AGENT_LINUX_ARM64
                                    image IMAGE_LINUX_DEB
                                    registryUrl 'https://public.ecr.aws/'
                                    alwaysPull true
                                }
                            }
                            steps {
                                script {
                                    def buildManifestObj = archiveAssembleUpload(
                                        componentName: "${COMPONENT_NAME}",
                                        inputManifest: "manifests/${INPUT_MANIFEST}",
                                        platform: 'linux',
                                        architecture: 'arm64',
                                        distribution: 'deb',
                                        continueOnError: params.CONTINUE_ON_ERROR,
                                        stashName: "build-archive-linux-arm64-deb-${JOB_NAME}-${BUILD_NUMBER}"
                                    )
                                    String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER)
                                    String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER)
                                    env.ARTIFACT_URL_LINUX_ARM64_DEB = artifactUrl
                                    env.INDEX_FILE_PATH_ARM64_DEB = buildManifestObj.getIndexFileRoot("${JOB_NAME}")

                                    echo "buildManifestUrl (linux, arm64, deb): ${buildManifestUrl}"
                                    echo "artifactUrl (linux, arm64, deb): ${artifactUrl}"

                                    String bundleManifestUrl = buildManifestObj.getBundleManifestUrl(JOB_NAME, BUILD_NUMBER)

                                    triggerIntegrationTests(buildManifestUrl, 'linux', 'deb')
                                }
                            }
                            post {
                                success {
                                    script {
                                        if (params.UPDATE_LATEST_URL) {
                                            uploadIndexFile(
                                                indexFilePath: env.INDEX_FILE_PATH_ARM64_DEB
                                            )
                                        }
                                    }
                                }
                                unstable {
                                    script {
                                        if (params.UPDATE_LATEST_URL) {
                                            uploadIndexFile(
                                                indexFilePath: env.INDEX_FILE_PATH_ARM64_DEB
                                            )
                                        }
                                    }
                                }
                                always {
                                    script {
                                        postCleanup()
                                    }
                                }
                            }
                        }
                    }
                }
                stage('build-and-test-windows-x64-zip') {
                    when {
                        beforeAgent true
                        expression {
                            params.BUILD_PLATFORM.contains('windows')
                        }
                        expression {
                            params.BUILD_DISTRIBUTION.contains('zip')
                        }
                    }
                    agent {
                        docker {
                            label AGENT_WINDOWS_X64
                            image IMAGE_WINDOWS_ZIP
                            registryUrl 'https://public.ecr.aws/'
                            alwaysPull true
                        }
                    }
                    steps {
                        script {
                            echo("Switching to Java ${env.javaVersionNumber} on Windows Docker Container")
                            sh("scoop reset `scoop list jdk | cut -d ' ' -f1 | grep ${env.javaVersionNumber} | head -1`")
                            def buildManifestObj = buildAssembleUpload(
                                componentName: "${COMPONENT_NAME}",
                                inputManifest: "manifests/${INPUT_MANIFEST}",
                                platform: 'windows',
                                architecture: 'x64',
                                distribution: 'zip',
                                continueOnError: params.CONTINUE_ON_ERROR,
                                incremental: params.INCREMENTAL,
                                previousBuildId: params.PREVIOUS_BUILD_ID
                            )
                            String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER)
                            String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER)
                            env.ARTIFACT_URL_LINUX_WINDOWS_X64_ZIP = artifactUrl
                            env.INDEX_FILE_PATH_X64_ZIP = buildManifestObj.getIndexFileRoot("${JOB_NAME}")

                            echo "buildManifestUrl (windows, x64, zip): ${buildManifestUrl}"
                            echo "artifactUrl (windows, x64, zip): ${artifactUrl}"

                            triggerIntegrationTests(buildManifestUrl, 'windows', 'zip')
                        }
                    }
                    post {
                        success {
                            script {
                                if (params.UPDATE_LATEST_URL) {
                                    uploadIndexFile(
                                        indexFilePath: env.INDEX_FILE_PATH_X64_ZIP
                                    )
                                }
                            }
                        }
                        unstable {
                            script {
                                if (params.UPDATE_LATEST_URL) {
                                    uploadIndexFile(
                                        indexFilePath: env.INDEX_FILE_PATH_X64_ZIP
                                    )
                                }
                            }
                        }
                        always {
                            script {
                                if (params.CONTINUE_ON_ERROR) {
                                    markStageUnstableIfPluginsFailedToBuild()
                                }
                                postCleanup()
                            }
                        }
                    }
                }
            }
        }
        stage('docker build') {
            when {
                beforeAgent true
                allOf {
                    expression {
                        params.BUILD_DOCKER != 'do_not_build_docker'
                    }
                    expression {
                        params.BUILD_PLATFORM.contains('linux')
                    }
                }
            }
            agent {
                docker {
                    label AGENT_LINUX_X64
                    image dockerAgent.image
                    args dockerAgent.args
                    registryUrl 'https://public.ecr.aws/'
                    alwaysPull true
                }
            }
            steps {
                script {
                    echo "env.ARTIFACT_URL_LINUX_X64_TAR: ${env.ARTIFACT_URL_X64_TAR}"
                    echo "env.ARTIFACT_URL_LINUX_ARM64_TAR: ${env.ARTIFACT_URL_ARM64_TAR}"

                    buildDockerImage(
                        inputManifest: "manifests/${INPUT_MANIFEST}",
                        buildNumber: "${BUILD_NUMBER}",
                        buildOption: "${BUILD_DOCKER}",
                        artifactUrlX64: env.ARTIFACT_URL_LINUX_X64_TAR,
                        artifactUrlArm64: env.ARTIFACT_URL_LINUX_ARM64_TAR
                    )
                }
            }
        }
    }
    post {
        always {
            node(AGENT_LINUX_X64) {
                checkout scm
                script {
                    sleep 10
                    def rc = (params.RC_NUMBER.toInteger() > 0)
                    publishDistributionBuildResults(
                        failureMessages: buildMessage(search: 'Error building'),
                        passMessages: buildMessage(search: 'Successfully built'),
                        inputManifestPath: "manifests/$INPUT_MANIFEST",
                        rc: rc,
                        rcNumber: env.RC_NUMBER,
                        componentCategory: "OpenSearch"
                    )
                    if (params.UPDATE_GITHUB_ISSUE) {
                        updateBuildFailureIssues(
                            inputManifestPath: "manifests/$INPUT_MANIFEST",
                            distributionBuildNumber: "${BUILD_NUMBER}"
                        )
                    }
                    postCleanup()
                }
            
            }
        }
        success {
            node(AGENT_LINUX_X64) {
                script {
                    postCleanup()
                }
            }
        }
        failure {
            node(AGENT_LINUX_X64) {
                checkout scm
                script {
                    postCleanup()
                }
            }
        }
    }
}

def markStageUnstableIfPluginsFailedToBuild() {
    def stageLogs = getLogsForStage(stageName: "${STAGE_NAME}")
    if (stageLogs.any { e -> e.contains('Failed plugins are') }) {
        unstable('Some plugins failed to build. See the ./build.sh step for logs and more details')
}
    }

def triggerIntegrationTests(String buildManifestUrl, String platform, String distribution) {
    Boolean skipIntegTests = (INTEG_TEST_JOB_NAME == '' || TEST_MANIFEST == '' || buildManifestUrl == '' || !TEST_PLATFORM.contains(platform) || !TEST_DISTRIBUTION.contains(distribution))
    echo "${skipIntegTests ? 'Skipping INTEG tests as one of the values has empty or wrong string: INTEG_TEST_JOB_NAME, TEST_MANIFEST, buildManifestUrl, TEST_PLATFORM, TEST_DISTRIBUTION' : 'Running INTEG tests'}"
    if (!skipIntegTests) {
        def integTestResults =
                build job: INTEG_TEST_JOB_NAME,
                propagate: false,
                wait: false,
                parameters: [
                    string(name: 'TEST_MANIFEST', value: TEST_MANIFEST),
                    string(name: 'BUILD_MANIFEST_URL', value: buildManifestUrl),
                    string(name: 'RC_NUMBER', value: RC_NUMBER),
                    booleanParam(name: 'UPDATE_GITHUB_ISSUES', value: true)
                ]
    }
}

def triggerBWCTests(String buildManifestUrl, String platform, String distribution) {
    Boolean skipBwcTests = (BWC_TEST_JOB_NAME == '' || TEST_MANIFEST == '' || buildManifestUrl == '' || !TEST_PLATFORM.contains(platform) || !TEST_DISTRIBUTION.contains(distribution))
    echo "${skipBwcTests ? 'Skipping BWC tests as one of the values has empty or wrong string: BWC_TEST_JOB_NAME, TEST_MANIFEST, buildManifestUrl, TEST_PLATFORM, TEST_DISTRIBUTION' : 'Running BWC tests'}"
    if (!skipBwcTests) {
        def bwcTestResults =
                build job: BWC_TEST_JOB_NAME,
                propagate: false,
                wait: false,
                parameters: [
                    string(name: 'TEST_MANIFEST', value: TEST_MANIFEST),
                    string(name: 'BUILD_MANIFEST_URL', value: buildManifestUrl),
                    string(name: 'AGENT_LABEL', value: AGENT_LINUX_X64)
                ]
    }
}

def verifyParameterPlatformDistribution(String paramName, String allowedValue) {
    echo("Verify Parameter '$paramName'")
    def paramValue = env."$paramName"
    def checkArray = paramValue.tokenize(' ')
    echo("Parameter Entry: '$paramValue', $checkArray")
    echo("Supported Entry: '$allowedValue'")

    if (paramValue == null || paramValue == '') {
        currentBuild.result = 'ABORTED'
        error("Missing parameter '$paramName' (possible entries: $allowedValue).")
    }

    for (String entry : paramValue) {
        if (! allowedValue.contains(entry)) {
            currentBuild.result = 'ABORTED'
            error("Error parameter '$paramName': $paramValue (possible entries: ${allowedValue}).")
        }
    }

    echo("Verified '$paramName': $paramValue\n")
}