From 744cf4f6e77ddd233210fc0e0e0e83177b363842 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 18 Oct 2022 13:59:34 -0400 Subject: [PATCH 1/3] Add platform selector to both check build and dashboards build Signed-off-by: Peter Zhu --- jenkins/check-for-build.jenkinsfile | 8 +- .../distribution-build.jenkinsfile | 129 ++++++++++++------ 2 files changed, 93 insertions(+), 44 deletions(-) diff --git a/jenkins/check-for-build.jenkinsfile b/jenkins/check-for-build.jenkinsfile index 1319926a2f..67af17ba45 100644 --- a/jenkins/check-for-build.jenkinsfile +++ b/jenkins/check-for-build.jenkinsfile @@ -40,6 +40,11 @@ pipeline { description: 'Job to trigger if build has changed', trim: true ) + string( + name: 'BUILD_PLATFORM', + description: 'Platform to build', + trim: true + ) } stages { stage('detect docker image + args') { @@ -81,7 +86,8 @@ pipeline { } build job: "${TARGET_JOB_NAME}", parameters: [ string(name: 'INPUT_MANIFEST', value: "${INPUT_MANIFEST}"), - string(name: 'TEST_MANIFEST', value: "${TEST_MANIFEST}") + string(name: 'TEST_MANIFEST', value: "${TEST_MANIFEST}"), + string(name: 'BUILD_PLATFORM', value: "${BUILD_PLATFORM}") ], wait: true echo "Build succeeded, uploading build SHA for that job" diff --git a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile index bb43ce96e3..97b7ea0dd4 100644 --- a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile +++ b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile @@ -41,6 +41,12 @@ pipeline { defaultValue: "bwc-test-opensearch-dashboards", trim: true ) + string( // Note: need to update 'verify-parameters' entries if you add new platform(s) + name: 'BUILD_PLATFORM', + description: "Build selected platform related artifacts, choices include 'linux', 'macos', 'windows'. Can combine multiple platforms with space in between (maven snapshot is only on linux)", + defaultValue: 'linux', + trim: true + ) choice( name: 'BUILD_DOCKER', description: 'Build docker image or not with options.', @@ -53,7 +59,7 @@ pipeline { ) } stages { - stage('detect docker image + args') { + stage('verify-parameters') { agent { docker { label AGENT_X64 @@ -62,9 +68,22 @@ pipeline { } } steps { - script { - currentBuild.description = "$INPUT_MANIFEST" + script { + echo("Detect Docker Images and Related Parameters") dockerAgent = detectDockerAgent() + currentBuild.description = INPUT_MANIFEST + + echo("Verify Build Platforms") + def build_platform_array = params.BUILD_PLATFORM.tokenize(' ') + echo("User Entry Platforms: '${params.BUILD_PLATFORM}', ${build_platform_array}") + def all_platforms = "linux" + echo("All Supported Platforms: '${all_platforms}'") + for (String plat : build_platform_array) { + if (! all_platforms.contains(plat)) { + currentBuild.result = 'ABORTED' + error("Missing parameter: BUILD_PLATFORM (possible entries: ${all_platforms}).") + } + } } } } @@ -95,6 +114,12 @@ pipeline { stage('build') { parallel { stage('build-and-test-linux-x64-tar') { + when { + beforeAgent true + expression{ + params.BUILD_PLATFORM.contains('linux') + } + } agent { docker { label AGENT_X64 @@ -114,11 +139,11 @@ pipeline { ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) - env.ARTIFACT_URL_X64_TAR = artifactUrl + env.ARTIFACT_URL_LINUX_X64_TAR = artifactUrl env.INDEX_FILE_PATH = buildManifestObj.getIndexFileRoot("${JOB_NAME}") - echo "buildManifestUrl (x64, tar): ${buildManifestUrl}" - echo "artifactUrl (x64, tar): ${artifactUrl}" + echo "buildManifestUrl (linux, x64, tar): ${buildManifestUrl}" + echo "artifactUrl (linux, x64, tar): ${artifactUrl}" parallel([ 'integ-test': { @@ -138,11 +163,11 @@ pipeline { buildInfoYaml( componentName: COMPONENT_NAME, status: integTestResults.getId(), - stage: "integ_test_x64" + stage: "integ_test_x64" // TODO: change to integ_test_linux_x64_tar ) - env.ARTIFACT_URL_X64_TAR_INTEG_TEST_RESULT = createTestResultsMessage( - testType: "Integ Tests (x64, tar)", + env.ARTIFACT_URL_LINUX_X64_TAR_INTEG_TEST_RESULT = createTestResultsMessage( + testType: "Integ Tests (linux, x64, tar)", status: integTestResults.getResult(), absoluteUrl: integTestResults.getAbsoluteUrl() ) @@ -165,11 +190,11 @@ pipeline { buildInfoYaml( componentName: COMPONENT_NAME, status: bwcTestResults.getId(), - stage: "bwc_test_x64" + stage: "bwc_test_x64" // TODO: change to bwc_test_linux_x64_tar ) - env.ARTIFACT_URL_X64_TAR_BWC_TEST_RESULT = createTestResultsMessage( - testType: "BWC Tests (x64, tar)", + env.ARTIFACT_URL_LINUX_X64_TAR_BWC_TEST_RESULT = createTestResultsMessage( + testType: "BWC Tests (linux, x64, tar)", status: bwcTestResults.getResult(), absoluteUrl: bwcTestResults.getAbsoluteUrl() ) @@ -184,8 +209,8 @@ pipeline { lib.jenkins.Messages.new(this).add( "${STAGE_NAME}", lib.jenkins.Messages.new(this).get(["${STAGE_NAME}"]) + - "\n${env.ARTIFACT_URL_X64_TAR_INTEG_TEST_RESULT}" + - "\n${env.ARTIFACT_URL_X64_TAR_BWC_TEST_RESULT}" + "\n${env.ARTIFACT_URL_LINUX_X64_TAR_INTEG_TEST_RESULT}" + + "\n${env.ARTIFACT_URL_LINUX_X64_TAR_BWC_TEST_RESULT}" ) postCleanup() @@ -194,6 +219,12 @@ pipeline { } } stage('build-and-test-linux-x64-rpm') { + when { + beforeAgent true + expression{ + params.BUILD_PLATFORM.contains('linux') + } + } agent { label AGENT_X64 } stages { stage('build-archive-linux-x64-rpm') { @@ -243,14 +274,14 @@ pipeline { ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) - env.ARTIFACT_URL_X64_RPM = artifactUrl + env.ARTIFACT_URL_LINUX_X64_RPM = artifactUrl - echo "buildManifestUrl (x64, rpm): ${buildManifestUrl}" - echo "artifactUrl (x64, rpm): ${artifactUrl}" + echo "buildManifestUrl (linux, x64, rpm): ${buildManifestUrl}" + echo "artifactUrl (linux, x64, rpm): ${artifactUrl}" String bundleManifestUrl = buildManifestObj.getBundleManifestUrl(JOB_NAME, BUILD_NUMBER) - echo "Trigger rpm validation for x64 rpm ${bundleManifestUrl}" + echo "Trigger rpm validation for linux x64 rpm ${bundleManifestUrl}" def rpmValidationResults = build job: 'rpm-validation', propagate: false, @@ -260,8 +291,8 @@ pipeline { string(name: 'AGENT_LABEL', value: AGENT_X64) ] - env.RPM_VALIDATION_X64_RPM_TEST_RESULT = createTestResultsMessage( - testType: "RPM Validation (x64, rpm)", + env.RPM_VALIDATION_LINUX_X64_RPM_TEST_RESULT = createTestResultsMessage( + testType: "RPM Validation (linux, x64, rpm)", status: rpmValidationResults.getResult(), absoluteUrl: rpmValidationResults.getAbsoluteUrl() ) @@ -273,7 +304,7 @@ pipeline { lib.jenkins.Messages.new(this).add( "${STAGE_NAME}", lib.jenkins.Messages.new(this).get(["${STAGE_NAME}"]) + - "\n${env.RPM_VALIDATION_X64_RPM_TEST_RESULT}" + "\n${env.RPM_VALIDATION_LINUX_X64_RPM_TEST_RESULT}" ) postCleanup() @@ -284,6 +315,12 @@ pipeline { } } stage('build-and-test-linux-arm64-tar') { + when { + beforeAgent true + expression{ + params.BUILD_PLATFORM.contains('linux') + } + } agent { label AGENT_X64 } stages { stage('build-archive-linux-arm64-tar') { @@ -335,9 +372,9 @@ pipeline { String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) - env.ARTIFACT_URL_ARM64_TAR = artifactUrl - echo "buildManifestUrl (arm64, tar): ${buildManifestUrl}" - echo "artifactUrl (arm64, tar): ${artifactUrl}" + env.ARTIFACT_URL_LINUX_ARM64_TAR = artifactUrl + echo "buildManifestUrl (linux, arm64, tar): ${buildManifestUrl}" + echo "artifactUrl (linux, arm64, tar): ${artifactUrl}" parallel([ 'integ-test': { @@ -357,11 +394,11 @@ pipeline { buildInfoYaml( componentName: COMPONENT_NAME, status: integTestResults.getId(), - stage: "integ_test_arm64" + stage: "integ_test_arm64" // TODO: change to integ_test_linux_arm64_tar ) - env.ARTIFACT_URL_ARM64_TAR_INTEG_TEST_RESULT = createTestResultsMessage( - testType: "Integ Tests (arm64, tar)", + env.ARTIFACT_URL_LINUX_ARM64_TAR_INTEG_TEST_RESULT = createTestResultsMessage( + testType: "Integ Tests (linux, arm64, tar)", status: integTestResults.getResult(), absoluteUrl: integTestResults.getAbsoluteUrl() ) @@ -384,11 +421,11 @@ pipeline { buildInfoYaml( componentName: COMPONENT_NAME, status: bwcTestResults.getId(), - stage: "bwc_test_arm64" + stage: "bwc_test_arm64" // TODO: change to bwc_test_linux_arm64_tar ) - env.ARTIFACT_URL_ARM64_TAR_BWC_TEST_RESULT = createTestResultsMessage( - testType: "BWC Tests (arm64, tar)", + env.ARTIFACT_URL_LINUX_ARM64_TAR_BWC_TEST_RESULT = createTestResultsMessage( + testType: "BWC Tests (linux, arm64, tar)", status: bwcTestResults.getResult(), absoluteUrl: bwcTestResults.getAbsoluteUrl() ) @@ -403,8 +440,8 @@ pipeline { lib.jenkins.Messages.new(this).add( "${STAGE_NAME}", lib.jenkins.Messages.new(this).get(["${STAGE_NAME}"]) + - "\n${env.ARTIFACT_URL_ARM64_TAR_INTEG_TEST_RESULT}" + - "\n${env.ARTIFACT_URL_ARM64_TAR_BWC_TEST_RESULT}" + "\n${env.ARTIFACT_URL_LINUX_ARM64_TAR_INTEG_TEST_RESULT}" + + "\n${env.ARTIFACT_URL_LINUX_ARM64_TAR_BWC_TEST_RESULT}" ) postCleanup() @@ -415,6 +452,12 @@ pipeline { } } stage('build-and-test-linux-arm64-rpm') { + when { + beforeAgent true + expression{ + params.BUILD_PLATFORM.contains('linux') + } + } agent { label AGENT_X64 } stages { stage('build-archive-linux-arm64-rpm') { @@ -464,13 +507,13 @@ pipeline { ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) - env.ARTIFACT_URL_ARM64_RPM = artifactUrl - echo "buildManifestUrl (arm64, rpm): ${buildManifestUrl}" - echo "artifactUrl (arm64, rpm): ${artifactUrl}" + env.ARTIFACT_URL_LINUX_ARM64_RPM = artifactUrl + echo "buildManifestUrl (linux, arm64, rpm): ${buildManifestUrl}" + echo "artifactUrl (linux, arm64, rpm): ${artifactUrl}" String bundleManifestUrl = buildManifestObj.getBundleManifestUrl(JOB_NAME, BUILD_NUMBER) - echo "Trigger rpm validation for arm64 rpm ${bundleManifestUrl}" + echo "Trigger rpm validation for linux arm64 rpm ${bundleManifestUrl}" def rpmValidationResults = build job: 'rpm-validation', propagate: false, @@ -480,8 +523,8 @@ pipeline { string(name: 'AGENT_LABEL', value: AGENT_ARM64) ] - env.RPM_VALIDATION_ARM64_RPM_TEST_RESULT = createTestResultsMessage( - testType: "RPM Validation (arm64, rpm)", + env.RPM_VALIDATION_LINUX_ARM64_RPM_TEST_RESULT = createTestResultsMessage( + testType: "RPM Validation (linux, arm64, rpm)", status: rpmValidationResults.getResult(), absoluteUrl: rpmValidationResults.getAbsoluteUrl() ) @@ -493,7 +536,7 @@ pipeline { lib.jenkins.Messages.new(this).add( "${STAGE_NAME}", lib.jenkins.Messages.new(this).get(["${STAGE_NAME}"]) + - "\n${env.RPM_VALIDATION_ARM64_RPM_TEST_RESULT}" + "\n${env.RPM_VALIDATION_LINUX_ARM64_RPM_TEST_RESULT}" ) postCleanup() @@ -539,15 +582,15 @@ pipeline { } steps { script { - echo "env.ARTIFACT_URL_X64_TAR: ${env.ARTIFACT_URL_X64_TAR}" - echo "env.ARTIFACT_URL_ARM64_TAR: ${env.ARTIFACT_URL_ARM64_TAR}" + echo "env.ARTIFACT_URL_LINUX_X64_TAR: ${env.ARTIFACT_URL_LINUX_X64_TAR}" + echo "env.ARTIFACT_URL_LINUX_ARM64_TAR: ${env.ARTIFACT_URL_LINUX_ARM64_TAR}" buildDockerImage( inputManifest: "manifests/${INPUT_MANIFEST}", buildNumber: "${BUILD_NUMBER}", buildOption: "${BUILD_DOCKER}", - artifactUrlX64: env.ARTIFACT_URL_X64_TAR, - artifactUrlArm64: env.ARTIFACT_URL_ARM64_TAR + artifactUrlX64: env.ARTIFACT_URL_LINUX_X64_TAR, + artifactUrlArm64: env.ARTIFACT_URL_LINUX_ARM64_TAR ) } } From 806011bac1400d538f303f97db1b1322c2214894 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 18 Oct 2022 14:13:49 -0400 Subject: [PATCH 2/3] Add platform selector to both check build and dashboards build Signed-off-by: Peter Zhu --- jenkins/opensearch-dashboards/distribution-build.jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile index 97b7ea0dd4..bc5186ebda 100644 --- a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile +++ b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile @@ -43,7 +43,7 @@ pipeline { ) string( // Note: need to update 'verify-parameters' entries if you add new platform(s) name: 'BUILD_PLATFORM', - description: "Build selected platform related artifacts, choices include 'linux', 'macos', 'windows'. Can combine multiple platforms with space in between (maven snapshot is only on linux)", + description: "Build selected platform related artifacts, choices include 'linux'. Can combine multiple platforms with space in between", defaultValue: 'linux', trim: true ) From 95d1f4ce5aa880c8867df78e8f4bdc7ee65d73f8 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 18 Oct 2022 14:17:35 -0400 Subject: [PATCH 3/3] Add platform selector to both check build and dashboards build Signed-off-by: Peter Zhu --- .../opensearch-dashboards/distribution-build.jenkinsfile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile index bc5186ebda..72fc1eceae 100644 --- a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile +++ b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile @@ -568,8 +568,13 @@ pipeline { stage('docker build') { when { beforeAgent true - expression { - params.BUILD_DOCKER != 'do_not_build_docker' + allOf{ + expression { + params.BUILD_DOCKER != 'do_not_build_docker' + } + expression{ + params.BUILD_PLATFORM.contains('linux') + } } } agent {