-
Notifications
You must be signed in to change notification settings - Fork 120
/
Copy pathJenkinsfile
220 lines (194 loc) · 12.3 KB
/
Jenkinsfile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
pipeline {
agent none
options {
disableConcurrentBuilds()
overrideIndexTriggers(false)
skipDefaultCheckout(true)
}
parameters {
// Allow job runner to filter based on platform
// Use the line below to enable all PW clusters
// choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use')
// Use the line below to enable the PW AWS cluster
// choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use')
// Use the line below to enable hera
// choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion'], description: 'Specify the platform(s) to use')
choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'jet', 'orion'], description: 'Specify the platform(s) to use')
// Allow job runner to filter based on compiler
choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build')
// Uncomment the following line to re-enable comprehensive tests
// booleanParam name: 'SRW_WE2E_COMPREHENSIVE_TESTS', defaultValue: false, description: 'Whether to execute the comprehensive end-to-end tests'
}
stages {
// Uncomment the following block to re-enable PW clusters
/*
// Start the NOAA Parallel Works clusters, if necessary
stage('Start Parallel Works Clusters') {
matrix {
// Start all clusters by default or only the specified cluster given by SRW_PLATFORM_FILTER
when {
anyOf {
expression { params.SRW_PLATFORM_FILTER == 'all' }
expression { params.SRW_PLATFORM_FILTER == env.SRW_PLATFORM }
}
}
axes {
axis {
name 'SRW_PLATFORM'
values 'pclusternoaav2use1' //, 'azclusternoaav2eus1', 'gclusternoaav2usc1'
}
}
stages {
// Call the parallel-works-jenkins-client/start-cluster job using SRW_PLATFORM for the
// PW_CLUSTER_NAME parameter
stage('Start Cluster') {
steps {
build job: 'parallel-works-jenkins-client/start-cluster', parameters: [string(name: 'PW_CLUSTER_NAME', value: env.SRW_PLATFORM), string(name: 'PW_CLUSTER_SSH_KEY', value: '~/.ssh/id_rsa'), string(name: 'JAVA_VERSION', value: '11')]
}
}
}
}
}
*/
// Build and test the SRW application on all supported platforms using the supported compilers for each platform
stage('Build and Test') {
matrix {
// Run on all platform/compiler combinations by default or build and test only on the platform(s) and
// compiler(s) specified by SRW_PLATFORM_FILTER and SRW_COMPILER_FILTER
when {
allOf {
anyOf {
expression { params.SRW_PLATFORM_FILTER == 'all' }
expression { params.SRW_PLATFORM_FILTER == env.SRW_PLATFORM }
}
anyOf {
expression { params.SRW_COMPILER_FILTER == 'all' }
expression { params.SRW_COMPILER_FILTER == env.SRW_COMPILER }
}
}
}
axes {
axis {
name 'SRW_PLATFORM'
// Uncomment line below to re-add use of Hera
// values 'cheyenne', 'gaea', 'hera', 'jet', 'orion' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'
values 'cheyenne', 'gaea', 'jet', 'orion' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'
}
axis {
name 'SRW_COMPILER'
values 'gnu', 'intel'
}
}
excludes {
// Exclude GNU from platforms that don't support it
exclude {
axis {
name 'SRW_PLATFORM'
values 'gaea', 'jet', 'orion' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1'
}
axis {
name 'SRW_COMPILER'
values 'gnu'
}
}
}
agent {
label env.SRW_PLATFORM
}
environment {
BRANCH_NAME_ESCAPED = env.BRANCH_NAME.replace('/', '_')
BUILD_VERSION = "${env.SRW_PLATFORM}-${env.SRW_COMPILER}-${env.BRANCH_NAME_ESCAPED}-${env.BUILD_NUMBER}"
BUILD_NAME = "ufs-srweather-app_${env.BUILD_VERSION}"
INSTALL_NAME = "install_${env.SRW_COMPILER}"
}
stages {
// Clean the workspace, checkout the repository, and run checkout_externals
stage('Initialize') {
steps {
echo "Initializing SRW (${env.SRW_COMPILER}) build environment on ${env.SRW_PLATFORM} (using ${env.WORKSPACE})"
cleanWs()
checkout scm
sh '"${WORKSPACE}/manage_externals/checkout_externals"'
}
}
// Run the unified build script; if successful create a tarball of the build and upload to S3
stage('Build') {
steps {
echo "Building SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM} (using ${env.WORKSPACE})"
sh 'bash --login "${WORKSPACE}/.cicd/scripts/srw_build.sh"'
}
post {
success {
sh 'cd "${WORKSPACE}/${INSTALL_NAME}" && tar --create --gzip --verbose --file "${WORKSPACE}/${BUILD_NAME}.tgz" *'
s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'woc-epic-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.BUILD_NAME}.tgz", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'woc-epic-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "build_${env.SRW_COMPILER}/srw_build-${env.SRW_PLATFORM}-${env.SRW_COMPILER}.log", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
}
}
}
// Run the unified test script
stage('Test') {
environment {
SRW_WE2E_EXPERIMENT_BASE_DIR = "${env.WORKSPACE}/expt_dirs"
}
steps {
echo "Testing SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM} (using ${env.WORKSPACE})"
// Remove the following line to re-enable comprehensive tests
sh 'SRW_WE2E_COMPREHENSIVE_TESTS=false bash --login "${WORKSPACE}/.cicd/scripts/srw_test.sh"'
// Uncomment the following block to re-enable comprehensive tests
/*
// If executing for a Pull Request, check for the run_we2e_comprehensive_tests. If set,
// override the value of the SRW_WE2E_COMPREHENSIVE_TESTS parameter
script {
def run_we2e_comprehensive_tests = params.SRW_WE2E_COMPREHENSIVE_TESTS
def run_we2e_comprehensive_tests_label = 'run_we2e_comprehensive_tests'
if (env.CHANGE_ID) {
pullRequest.labels.each {
if (it == run_we2e_comprehensive_tests_label) {
run_we2e_comprehensive_tests = true
}
}
}
sh "SRW_WE2E_COMPREHENSIVE_TESTS=${run_we2e_comprehensive_tests}" + ' bash --login "${WORKSPACE}/.cicd/scripts/srw_test.sh"'
}
*/
}
post {
always {
// Archive the test log files
sh 'cd "${SRW_WE2E_EXPERIMENT_BASE_DIR}" && tar --create --gzip --verbose --dereference --file "${WORKSPACE}/we2e_test_logs-${SRW_PLATFORM}-${SRW_COMPILER}.tgz" */log.generate_FV3LAM_wflow */log.launch_FV3LAM_wflow */log/*'
// Remove the data sets from the experiments directory to conserve disk space
sh 'find "${SRW_WE2E_EXPERIMENT_BASE_DIR}" -regextype posix-extended -regex "^.*(orog|[0-9]{10})$" -type d | xargs rm -rf'
s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'woc-epic-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: 'we2e_test_results-*-*.txt', storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'woc-epic-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: 'we2e_test_logs-*-*.tgz', storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
}
}
}
}
}
}
}
// Uncomment the following block to re-enable PW clusters
/*
post {
always {
// Stop any Parallel Works clusters that were started during the pipeline execution
script {
// def pw_clusters = ['pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1']
def pw_clusters = ['pclusternoaav2use1']
def clusters = []
// Determine which clusters need to be stopped, if any
if (params.SRW_PLATFORM_FILTER == 'all') {
clusters = pw_clusters
} else if (params.SRW_PLATFORM_FILTER in pw_clusters) {
clusters = [params.SRW_PLATFORM_FILTER]
} else {
echo 'No Parallel Works clusters were used in build'
}
for (int i = 0; i < clusters.size(); ++i) {
// Call the parallel-works-jenkins-client/stop-cluster job using clusters[i] for the
// PW_CLUSTER_NAME parameter
build job: 'parallel-works-jenkins-client/stop-cluster', parameters: [string(name: 'PW_CLUSTER_NAME', value: clusters[i])]
}
}
}
}
*/
}