-
Notifications
You must be signed in to change notification settings - Fork 109
/
code.pipeline.yml
336 lines (309 loc) · 10.1 KB
/
code.pipeline.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
##
# Code pipeline
##
#
# Main Buildkite pipeline for running code-related linters and tests.
#
docker_plugin_default_config: &docker_plugin_default_config
image: "oasisprotocol/oasis-core-ci:${DOCKER_OASIS_CORE_CI_BASE_TAG}"
always_pull: true
workdir: /workdir
volumes:
- /var/lib/buildkite-agent/.coveralls:/root/.coveralls
- /var/lib/buildkite-agent/.codecov:/root/.codecov
# IAS Development API keys.
- /var/lib/buildkite-agent/.oasis-ias:/root/.oasis-ias
# Shared Rust incremental compile caches.
- /var/tmp/cargo_ic/debug:/var/tmp/artifacts/default/debug/incremental
- /var/tmp/cargo_ic/debug_sgx:/var/tmp/artifacts/sgx/x86_64-unknown-linux-sgx/debug/incremental
# Shared Rust package checkouts directory.
- /var/tmp/cargo_pkg/git:/root/.cargo/git
- /var/tmp/cargo_pkg/registry:/root/.cargo/registry
# Shared Rust SGX standard library artifacts cache.
- /var/tmp/xargo_cache:/root/.xargo
# Shared Go package checkouts directory.
- /var/tmp/go_pkg:/root/go/pkg
# Intel SGX Application Enclave Services Manager (AESM) daemon running on
# the Buildkite host.
- /var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket
# Propapage the tmpfs mount.
- /tmp:/tmp
# NOTE: When changing the environment variables below, also copy the changes
# to the docker_plugin_sgx_config.
environment:
- "LC_ALL=C.UTF-8"
- "LANG=C.UTF-8"
- "CARGO_TARGET_DIR=/var/tmp/artifacts"
- "CARGO_INSTALL_ROOT=/root/.cargo"
- "CARGO_INCREMENTAL=0"
- "GOPROXY=https://proxy.golang.org/"
propagate-environment: true
unconfined: true
docker_plugin_sgx_config: &docker_plugin_sgx_config
<<: *docker_plugin_default_config
devices:
# Intel SGX device.
- /dev/isgx
environment:
- "OASIS_TEE_HARDWARE=intel-sgx"
# Copy of environment variables defined in docker_plugin_default_config.
- "LC_ALL=C.UTF-8"
- "LANG=C.UTF-8"
- "CARGO_TARGET_DIR=/var/tmp/artifacts"
- "CARGO_INSTALL_ROOT=/root/.cargo"
- "CARGO_INCREMENTAL=0"
- "GOPROXY=https://proxy.golang.org/"
docker_plugin: &docker_plugin
oasislabs/docker#v3.0.1-oasis1:
<<: *docker_plugin_default_config
docker_plugin_sgx: &docker_plugin_sgx
oasislabs/docker#v3.0.1-oasis1:
<<: *docker_plugin_sgx_config
retry: &retry_agent_failure
automatic:
- exit_status: 125 # ERRO[0092] error waiting for container: unexpected EOF
limit: 2
steps:
###########
# Lint jobs
###########
- label: Lint Go node
command:
- .buildkite/go/lint.sh
retry:
<<: *retry_agent_failure
plugins:
<<: *docker_plugin
- label: Lint Rust crate versions
command: .buildkite/rust/lint_crate_versions.sh
retry:
<<: *retry_agent_failure
plugins:
<<: *docker_plugin
- label: Audit Rust dependencies for vulnerabilities
command: .buildkite/rust/cargo_audit.sh
retry:
<<: *retry_agent_failure
plugins:
<<: *docker_plugin
- label: Audit Go dependencies for vulnerabilities
command: .buildkite/go/nancy_audit.sh
retry:
<<: *retry_agent_failure
plugins:
<<: *docker_plugin
############
# Build jobs
############
- label: Build Go node
command:
- .buildkite/go/build.sh
- .buildkite/go/upload_artifacts.sh
retry:
<<: *retry_agent_failure
plugins:
<<: *docker_plugin
- label: Build Rust runtime loader
command:
- .buildkite/rust/build_generic.sh /workdir -p oasis-core-runtime-loader
- .buildkite/rust/build_generic.sh /workdir -p test-long-term-client
- .buildkite/rust/build_generic.sh /workdir -p simple-keyvalue-client
- .buildkite/rust/build_generic.sh /workdir -p simple-keyvalue-enc-client
- .buildkite/rust/build_generic.sh /workdir -p simple-keyvalue-ops-client
# Upload the built artifacts.
- cd /var/tmp/artifacts/default/debug
- buildkite-agent artifact upload oasis-core-runtime-loader
# Clients for E2E tests.
- buildkite-agent artifact upload test-long-term-client
- buildkite-agent artifact upload simple-keyvalue-client
- buildkite-agent artifact upload simple-keyvalue-enc-client
- buildkite-agent artifact upload simple-keyvalue-ops-client
retry:
<<: *retry_agent_failure
plugins:
<<: *docker_plugin
####################
# Runtime build jobs
####################
- label: Build key manager runtime
command:
- .buildkite/rust/build_runtime.sh tests/runtimes/simple-keymanager
- .buildkite/rust/build_runtime.sh tests/runtimes/simple-keyvalue
# Upload the built artifacts.
- cd /var/tmp/artifacts/sgx/x86_64-fortanix-unknown-sgx/debug
- buildkite-agent artifact upload simple-keymanager.sgxs
- buildkite-agent artifact upload simple-keyvalue.sgxs
- buildkite-agent artifact upload simple-keyvalue-upgrade.sgxs
- buildkite-agent artifact upload simple-keymanager-upgrade.sgxs
- cd /var/tmp/artifacts/default/debug
- buildkite-agent artifact upload simple-keymanager
- buildkite-agent artifact upload simple-keyvalue
- buildkite-agent artifact upload simple-keyvalue-upgrade
- buildkite-agent artifact upload simple-keymanager-upgrade
retry:
<<: *retry_agent_failure
plugins:
<<: *docker_plugin
###########
# Test jobs
###########
- label: Test Rust crates
command:
# Build storage interoperability test helpers first.
- make build-helpers
- export OASIS_STORAGE_PROTOCOL_SERVER_BINARY=$(realpath go/storage/mkvs/interop/mkvs-test-helpers)
- .buildkite/rust/test_generic.sh .
retry:
<<: *retry_agent_failure
plugins:
<<: *docker_plugin
# Wait for all jobs defined before this point
# to finish running in parallel before continuing.
- wait
###################################################
# Test that build-fuzz still works (only on master)
###################################################
- label: Test fuzz builds
branches: master
command:
# TODO: Consider making this a part of the development Docker image.
- go get -u github.com/dvyukov/go-fuzz/go-fuzz github.com/dvyukov/go-fuzz/go-fuzz-build
- make -C go build-fuzz
retry:
<<: *retry_agent_failure
plugins:
<<: *docker_plugin
#####################################
# Test jobs requiring build artifacts
#####################################
- label: Test Go node
command:
- .buildkite/go/test_and_coverage.sh
artifact_paths:
- coverage-*.txt
- /tmp/oasis-node-test_*/test-node.log
retry:
<<: *retry_agent_failure
plugins:
<<: *docker_plugin
###############
# E2E test jobs
###############
- label: E2E tests
parallelism: 7
timeout_in_minutes: 30
command:
- .buildkite/scripts/download_e2e_test_artifacts.sh
- .buildkite/scripts/test_e2e.sh
artifact_paths:
- coverage-merged-e2e-*.txt
- /tmp/e2e/**/*.log
env:
OASIS_E2E_COVERAGE: enable
TEST_BASE_DIR: /tmp
# libp2p logging.
IPFS_LOGGING: debug
retry:
<<: *retry_agent_failure
plugins:
<<: *docker_plugin
###########################
# E2E test jobs - intel-sgx
###########################
- label: E2E tests - intel-sgx
parallelism: 5
timeout_in_minutes: 36
command:
- .buildkite/scripts/download_e2e_test_artifacts.sh
# Only run runtime scenarios as others do not use SGX.
- .buildkite/scripts/test_e2e.sh --scenario e2e/runtime/.*
artifact_paths:
- coverage-merged-e2e-*.txt
- /tmp/e2e/**/*.log
env:
OASIS_E2E_COVERAGE: enable
OASIS_EXCLUDE_E2E: e2e/runtime/txsource-multi,e2e/runtime/txsource-multi-short
TEST_BASE_DIR: /tmp
# libp2p logging.
IPFS_LOGGING: debug
agents:
queue: intel-sgx
retry:
<<: *retry_agent_failure
plugins:
<<: *docker_plugin_sgx
###############################
# E2E test - intel-sgx with IAS
###############################
- label: E2E tests - intel-sgx - IAS
timeout_in_minutes: 10
command:
- .buildkite/scripts/sgx_ias_tests.sh
# A unique string to identify the step. The value is available in the
# BUILDKITE_STEP_KEY and is used to ensure the generated coverage file
# names are unique across this pipeline.
key: sgx-ias
artifact_paths:
- coverage-merged-e2e-*.txt
- /tmp/e2e/**/*.log
env:
OASIS_E2E_COVERAGE: enable
TEST_BASE_DIR: /tmp
# libp2p logging.
IPFS_LOGGING: debug
agents:
queue: intel-sgx
retry:
<<: *retry_agent_failure
plugins:
<<: *docker_plugin_sgx
###############
# Consensus simulator tests
###############
- label: Consensus simulator tests
timeout_in_minutes: 3
command:
- .buildkite/scripts/download_e2e_test_artifacts.sh
- .buildkite/scripts/test_consim.sh
artifact_paths:
- /tmp/consim-datadir/*.json
- /tmp/consim-datadir/consim.log
env:
TEST_BASE_DIR: /tmp
retry:
<<: *retry_agent_failure
plugins:
<<: *docker_plugin
####################################
# Rust coverage job (only on master)
####################################
- label: Coverage Rust crates
branches: master
command:
# Build storage interoperability test helpers first.
- make build-helpers
- export OASIS_STORAGE_PROTOCOL_SERVER_BINARY=$(realpath go/storage/mkvs/interop/mkvs-test-helpers)
- .buildkite/rust/coverage.sh
# Don't cause the build to fail, as tarpaulin is pretty unstable at the moment.
soft_fail: true
retry:
<<: *retry_agent_failure
plugins:
<<: *docker_plugin
# Wait for all jobs defined before this point
# to finish running in parallel before continuing.
- wait
###########################
# Merge coverage and upload
###########################
- label: "Merge and upload coverage"
command:
- .buildkite/scripts/merge_coverage.sh
- .buildkite/scripts/upload_coverage.sh
artifact_paths:
- merged-coverage.txt
soft_fail: true
retry:
<<: *retry_agent_failure
plugins:
<<: *docker_plugin