forked from etcd-io/etcd
-
Notifications
You must be signed in to change notification settings - Fork 0
/
test
executable file
·612 lines (513 loc) · 18.9 KB
/
test
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
#!/usr/bin/env bash
#
# Run all etcd tests
# ./test
# ./test -v
#
#
# Run specified test pass
#
# $ PASSES=unit ./test
# $ PASSES=integration ./test
#
#
# Run tests for one package
# Each pass has different default timeout, if you just run tests in one package or 1 test case then you can set TIMEOUT
# flag for different expectation
#
# $ PASSES=unit PKG=./wal TIMEOUT=1m ./test
# $ PASSES=integration PKG=./clientv3 TIMEOUT=1m ./test
#
# Run specified unit tests in one package
# To run all the tests with prefix of "TestNew", set "TESTCASE=TestNew ";
# to run only "TestNew", set "TESTCASE="\bTestNew\b""
#
# $ PASSES=unit PKG=./wal TESTCASE=TestNew TIMEOUT=1m ./test
# $ PASSES=unit PKG=./wal TESTCASE="\bTestNew\b" TIMEOUT=1m ./test
# $ PASSES=integration PKG=./client/integration TESTCASE="\bTestV2NoRetryEOF\b" TIMEOUT=1m ./test
#
#
# Run code coverage
# COVERDIR must either be a absolute path or a relative path to the etcd root
# $ COVERDIR=coverage PASSES="build build_cov cov" ./test
# $ go tool cover -html ./coverage/cover.out
set -e
set -o pipefail
# Consider command as failed when any component of the pipe fails:
# https://stackoverflow.com/questions/1221833/pipe-output-and-capture-exit-status-in-bash
set -o pipefail
# The test script is not supposed to make any changes to the files
# e.g. add/update missing dependencies. Such divergences should be
# detected and trigger a failure that needs explicit developer's action.
export GOFLAGS=-mod=readonly
source ./scripts/test_lib.sh
source ./build
PASSES=${PASSES:-"fmt bom dep build unit"}
PKG=${PKG:-}
if [ -z "$GOARCH" ]; then
GOARCH=$(go env GOARCH);
fi
# determine the number of CPUs to use for Go tests
CPU=${CPU:-"4"}
# determine whether target supports race detection
if [ -z "${RACE}" ] ; then
if [ "$GOARCH" == "amd64" ]; then
RACE="--race"
else
RACE="--race=false"
fi
else
RACE="--race=${RACE:-true}"
fi
# This options make sense for cases where SUT (System Under Test) is compiled by test.
COMMON_TEST_FLAGS=("-cpu=${CPU}" "${RACE}")
log_callout "Running with ${COMMON_TEST_FLAGS[*]}"
RUN_ARG=()
if [ -n "${TESTCASE}" ]; then
RUN_ARG=("-run=${TESTCASE}")
fi
function build_pass {
log_callout "Building etcd"
run_for_modules run go build "${@}" || return 2
GO_BUILD_FLAGS="-v" etcd_build "${@}"
GO_BUILD_FLAGS="-v" tools_build "${@}"
}
################# REGULAR TESTS ################################################
# run_unit_tests [pkgs] runs unit tests for a current module and givesn set of [pkgs]
function run_unit_tests {
local pkgs="${1:-./...}"
shift 1
# shellcheck disable=SC2086
go_test "${pkgs}" "parallel" : -short -timeout="${TIMEOUT:-3m}" "${COMMON_TEST_FLAGS[@]}" "${RUN_ARG[@]}" "$@"
}
function unit_pass {
run_for_modules run_unit_tests "$@"
}
function integration_extra {
if [ -z "${PKG}" ] ; then
run_for_module "." go_test "./contrib/raftexample" "keep_going" : -timeout="${TIMEOUT:-5m}" "${RUN_ARG[@]}" "${COMMON_TEST_FLAGS[@]}" "$@" || return $?
run_for_module "tests" go_test "./integration/v2store/..." "keep_going" : -tags v2v3 -timeout="${TIMEOUT:-5m}" "${RUN_ARG[@]}" "${COMMON_TEST_FLAGS[@]}" "$@" || return $?
else
log_warning "integration_extra ignored when PKG is specified"
fi
}
function integration_pass {
local pkgs=${USERPKG:-"./integration/..."}
run_for_module "tests" go_test "${pkgs}" "keep_going" : -timeout="${TIMEOUT:-30m}" "${COMMON_TEST_FLAGS[@]}" "${RUN_ARG[@]}" "$@" || return $?
integration_extra "$@"
}
function e2e_pass {
# e2e tests are running pre-build binary. Settings like --race,-cover,-cpu does not have any impact.
run_for_module "tests" go_test "./e2e/..." "keep_going" : -timeout="${TIMEOUT:-30m}" "${RUN_ARG[@]}" "$@"
}
function integration_e2e_pass {
run_pass "integration" "${@}"
run_pass "e2e" "${@}"
}
# generic_checker [cmd...]
# executes given command in the current module, and clearly fails if it
# failed or returned output.
function generic_checker {
local cmd=("$@")
if ! output=$("${cmd[@]}"); then
echo "${output}"
log_error -e "FAIL: '${cmd[*]}' checking failed (!=0 return code)"
return 255
fi
if [ -n "${output}" ]; then
echo "${output}"
log_error -e "FAIL: '${cmd[*]}' checking failed (printed output)"
return 255
fi
}
function functional_pass {
run ./tests/functional/build
# Clean up any data and logs from previous runs
rm -rf /tmp/etcd-functional-* /tmp/etcd-functional-*.backup
# TODO: These ports should be dynamically allocated instead of hard-coded.
for a in 1 2 3; do
./bin/etcd-agent --network tcp --address 127.0.0.1:${a}9027 < /dev/null &
pid="$!"
agent_pids="${agent_pids} $pid"
done
for a in 1 2 3; do
log_callout "Waiting for 'etcd-agent' on ${a}9027..."
while ! nc -z localhost ${a}9027; do
sleep 1
done
done
log_callout "functional test START!"
run ./bin/etcd-tester --config ./tests/functional/functional.yaml && log_success "'etcd-tester' succeeded"
local etcd_tester_exit_code=$?
if [[ "${etcd_tester_exit_code}" -ne "0" ]]; then
log_error "ETCD_TESTER_EXIT_CODE:" ${etcd_tester_exit_code}
fi
# shellcheck disable=SC2206
agent_pids=($agent_pids)
kill -s TERM "${agent_pids[@]}" || true
if [[ "${etcd_tester_exit_code}" -ne "0" ]]; then
log_error -e "\nFAILED! 'tail -1000 /tmp/etcd-functional-1/etcd.log'"
tail -1000 /tmp/etcd-functional-1/etcd.log
log_error -e "\nFAILED! 'tail -1000 /tmp/etcd-functional-2/etcd.log'"
tail -1000 /tmp/etcd-functional-2/etcd.log
log_error -e "\nFAILED! 'tail -1000 /tmp/etcd-functional-3/etcd.log'"
tail -1000 /tmp/etcd-functional-3/etcd.log
log_error "--- FAIL: exit code" ${etcd_tester_exit_code}
return ${etcd_tester_exit_code}
fi
log_success "functional test PASS!"
}
function grpcproxy_pass {
run_for_module "tests" go_test "./integration/... ./e2e" "fail_fast" : \
-timeout=30m -tags cluster_proxy "${COMMON_TEST_FLAGS[@]}" "$@"
}
################# COVERAGE #####################################################
# Builds artifacts used by tests/e2e in coverage mode.
function build_cov_pass {
local out="${BINDIR:-./bin}"
run go test -tags cov -c -covermode=set -coverpkg="./..." -o "${out}/etcd_test"
run go test -tags cov -c -covermode=set -coverpkg="./..." -o "${out}/etcdctl_test" "./etcdctl"
}
# pkg_to_coverflag [prefix] [pkgs]
# produces name of .coverprofile file to be used for tests of this package
function pkg_to_coverprofileflag {
local prefix="${1}"
local pkgs="${2}"
local pkgs_normalized
pkgs_normalized=$(echo "${pkgs}" | tr "./ " "__+")
echo -n "-coverprofile=${coverdir}/${prefix}_${pkgs_normalized}.coverprofile"
}
function cov_pass {
# shellcheck disable=SC2153
if [ -z "$COVERDIR" ]; then
log_error "COVERDIR undeclared"
return 255
fi
if [ ! -f "bin/etcd_test" ]; then
log_error "etcd_test binary not found. Call: PASSES='build_cov' ./test"
return 255
fi
local coverdir
coverdir=$(readlink -f "${COVERDIR}")
mkdir -p "${coverdir}"
rm -f "${coverdir}/*.coverprofile" "${coverdir}/cover.*"
local covpkgs
covpkgs=$(pkgs_in_module "./...")
local coverpkg_comma
coverpkg_comma=$(echo "${covpkgs[@]}" | xargs | tr ' ' ',')
local gocov_build_flags=("-covermode=set" "-coverpkg=$coverpkg_comma")
local failed=""
log_callout "Collecting coverage from unit tests ..."
go_test "./..." "keep_going" "pkg_to_coverprofileflag unit" -short -timeout=30m \
"${gocov_build_flags[@]}" "$@" || failed="$failed unit"
log_callout "Collecting coverage from integration tests ..."
run_for_module "tests" go_test "./integration/..." "keep_going" "pkg_to_coverprofileflag integration" \
-timeout=30m "${gocov_build_flags[@]}" "$@" || failed="$failed integration"
# integration-store-v2
run_for_module "tests" go_test "./integration/v2store/..." "keep_going" "pkg_to_coverprofileflag store_v2" \
-tags v2v3 -timeout=5m "${gocov_build_flags[@]}" "$@" || failed="$failed integration_v2v3"
# integration_cluster_proxy
run_for_module "tests" go_test "./integration/..." "keep_going" "pkg_to_coverprofileflag integration_cluster_proxy" \
-tags cluster_proxy -timeout=5m "${gocov_build_flags[@]}" || failed="$failed integration_cluster_proxy"
log_callout "Collecting coverage from e2e tests ..."
# We don't pass 'gocov_build_flags' nor 'pkg_to_coverprofileflag' here,
# as the coverage is colleced from the ./bin/etcd_test & ./bin/etcdctl_test internally spawned.
run_for_module "tests" go_test "./e2e/..." "keep_going" : -tags=cov -timeout 30m "$@" || failed="$failed tests_e2e"
log_callout "Collecting coverage from e2e tests with proxy ..."
run_for_module "tests" go_test "./e2e/..." "keep_going" : -tags="cov cluster_proxy" -timeout 30m "$@" || failed="$failed tests_e2e_proxy"
log_callout "Merging coverage results ..."
local cover_out_file="${coverdir}/cover.out"
# gocovmerge requires not-empty test to start with:
echo "mode: set" > "${cover_out_file}"
# incrementally merge to get coverage data even if some coverage files are corrupted
for f in "${coverdir}"/*.coverprofile; do
echo "merging test coverage file ${f}"
run_go_tool "github.com/gyuho/gocovmerge" "${f}" "${cover_out_file}" > "${coverdir}/cover.tmp" || failed="$failed gocovmerge:$f"
if [ -s "${coverdir}"/cover.tmp ]; then
mv "${coverdir}/cover.tmp" "${cover_out_file}"
fi
done
# strip out generated files (using GNU-style sed)
sed --in-place '/generated.go/d' "${cover_out_file}" || true
# held failures to generate the full coverage file, now fail
if [ -n "$failed" ]; then
for f in $failed; do
log_error "--- FAIL:" "$f"
done
log_warning "Despite failures, you can see partial report:"
log_warning " go tool cover -html ${cover_out_file}"
return 255
fi
log_success "done :) [see report: go tool cover -html ${cover_out_file}]"
}
######### Code formatting checkers #############################################
function fmt_pass {
toggle_failpoints disable
# TODO: add "unparam","staticcheck", "unconvert", "ineffasign","nakedret"
# after resolving ore-existing errors.
for p in shellcheck \
markdown_you \
goword \
gofmt \
govet \
revive \
license_header \
receiver_name \
mod_tidy \
dep \
shellcheck \
shellws \
; do
run_pass "${p}" "${@}"
done
}
function shellcheck_pass {
if tool_exists "shellcheck" "https://github.com/koalaman/shellcheck#installing"; then
generic_checker run shellcheck -fgcc build test scripts/*.sh
fi
}
function shellws_pass {
log_callout "Ensuring no tab-based indention in shell scripts"
local files
files=$(find ./ -name '*.sh' -print0 | xargs -0 )
log_cmd "grep -E -n $'^ *\t' ${files}"
# shellcheck disable=SC2086
if grep -E -n $'^ *\t' ${files} | sed -s $'s|\t|[\\\\tab]|g'; then
log_error "FAIL: found tab-based indention in bash scripts. Use ' ' (double space)."
return 1
else
log_success "SUCCESS: no tabulators found."
return 0
fi
}
function markdown_you_find_eschew_you {
local find_you_cmd="find . -name \*.md ! -path '*/vendor/*' ! -path './Documentation/*' ! -path './gopath.proto/*' ! -path './release/*' -exec grep -E --color '[Yy]ou[r]?[ '\''.,;]' {} + || true"
run eval "${find_you_cmd}"
}
function markdown_you_pass {
generic_checker markdown_you_find_eschew_you
}
function markdown_marker_pass {
# TODO: check other markdown files when marker handles headers with '[]'
if tool_exists "marker" "https://crates.io/crates/marker"; then
generic_checker run marker --skip-http --root ./Documentation 2>&1
fi
}
function govet_pass {
run_for_modules generic_checker run go vet
}
function govet_shadow_pass {
local shadow
shadow=$(tool_get_bin "golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow")
run_for_modules generic_checker run go vet -all -vettool="${shadow}"
}
function unparam_pass {
run_for_modules generic_checker run_go_tool "mvdan.cc/unparam"
}
function staticcheck_pass {
run_for_modules generic_checker run_go_tool "honnef.co/go/tools/cmd/staticcheck"
}
function revive_pass {
run_for_modules generic_checker run_go_tool "github.com/mgechev/revive" -config "${ETCD_ROOT_DIR}/tests/revive.toml" -exclude "vendor/..."
}
function unconvert_pass {
run_for_modules generic_checker run_go_tool "github.com/mdempsky/unconvert" unconvert -v
}
function ineffassign_per_package {
mapfile -t gofiles < <(go_srcs_in_module "$1")
run_go_tool github.com/gordonklaus/ineffassign "${gofiles[@]}"
}
function ineffassign_pass {
run_for_modules generic_checker ineffassign_per_package
}
function nakedret_pass {
run_for_modules generic_checker run_go_tool "github.com/alexkohler/nakedret"
}
function license_header_pass {
mapfile -t gofiles < <(go_srcs_in_module "$1")
for file in "${gofiles[@]}"; do
if ! head -n3 "${file}" | grep -Eq "(Copyright|generated|GENERATED)" ; then
licRes="${licRes}"$(echo -e " ${file}")
fi
done
if [ -n "${licRes}" ]; then
log_error -e "license header checking failed:\\n${licRes}"
return 255
fi
}
function receiver_name_for_package {
mapfile -t gofiles < <(go_srcs_in_module "$1")
recvs=$(grep 'func ([^*]' "${gofiles[@]}" | tr ':' ' ' | \
awk ' { print $2" "$3" "$4" "$1 }' | sed "s/[a-zA-Z\.]*go//g" | sort | uniq | \
grep -Ev "(Descriptor|Proto|_)" | awk ' { print $3" "$4 } ' | sort | uniq -c | grep -v ' 1 ' | awk ' { print $2 } ')
if [ -n "${recvs}" ]; then
# shellcheck disable=SC2206
recvs=($recvs)
for recv in "${recvs[@]}"; do
log_error "Mismatched receiver for $recv..."
grep "$recv" "${gofiles[@]}" | grep 'func ('
done
return 255
fi
}
function receiver_name_pass {
run_for_modules receiver_name_for_package
}
# goword_for_package package
# checks spelling and comments in the 'package' in the current module
#
function goword_for_package {
mapfile -t gofiles < <(go_srcs_in_module "$1")
local gowordRes
# spellchecking can be enabled with GOBINARGS="--tags=spell"
# but it requires heavy dependencies installation, like:
# apt-get install libaspell-dev libhunspell-dev hunspell-en-us aspell-en
# only check for broke exported godocs
if gowordRes=$(run_go_tool "github.com/chzchzchz/goword" -use-spell=false "${gofiles[@]}" | grep godoc-export | sort); then
log_error -e "goword checking failed:\\n${gowordRes}"
return 255
fi
if [ -n "$gowordRes" ]; then
log_error -e "goword checking returned output:\\n${gowordRes}"
return 255
fi
}
function goword_pass {
run_for_modules goword_for_package || return 255
}
function go_fmt_for_package {
# We utilize 'go fmt' to find all files suitable for formatting,
# but reuse full power gofmt to perform just RO check.
go fmt -n "$1" | sed 's| -w | -d |g' | sh
}
function gofmt_pass {
run_for_modules generic_checker go_fmt_for_package
}
function bom_pass {
log_callout "Checking bill of materials..."
# https://github.com/golang/go/commit/7c388cc89c76bc7167287fb488afcaf5a4aa12bf
# shellcheck disable=SC2207
modules=($(modules_exp))
# Internally license-bill-of-materials tends to modify go.sum
run cp go.sum go.sum.tmp || return 2
run cp go.mod go.mod.tmp || return 2
output=$(GOFLAGS=-mod=mod run_go_tool github.com/coreos/license-bill-of-materials \
--override-file ./bill-of-materials.override.json \
"${modules[@]}")
code="$?"
run cp go.sum.tmp go.sum || return 2
run cp go.mod.tmp go.mod || return 2
if [ "${code}" -ne 0 ] ; then
log_error -e "license-bill-of-materials (code: ${code}) failed with:\n${output}"
return 255
else
echo "${output}" > "bom-now.json.tmp"
fi
if ! diff ./bill-of-materials.json bom-now.json.tmp; then
log_error "modularized licenses do not match given bill of materials"
return 255
fi
rm bom-now.json.tmp
}
######## VARIOUS CHECKERS ######################################################
function dump_deps_of_module() {
local module
if ! module=$(run go list -m); then
return 255
fi
run go list -f "{{if not .Indirect}}{{if .Version}}{{.Path}},{{.Version}},${module}{{end}}{{end}}" -m all
}
# Checks whether dependencies are consistent across modules
function dep_pass {
local all_dependencies
all_dependencies=$(run_for_modules dump_deps_of_module | sort) || return 2
local duplicates
duplicates=$(echo "${all_dependencies}" | cut -d ',' -f 1,2 | sort | uniq | cut -d ',' -f 1 | sort | uniq -d) || return 2
for dup in ${duplicates}; do
log_error "FAIL: inconsistent versions for depencency: ${dup}"
echo "${all_dependencies}" | grep "${dup}" | sed "s|\([^,]*\),\([^,]*\),\([^,]*\)| - \1@\2 from: \3|g"
done
if [[ -n "${duplicates}" ]]; then
log_error "FAIL: inconsistent dependencies"
return 2
else
log_success "SUCCESS: dependencies are consistent across modules"
fi
}
function release_pass {
rm -f ./bin/etcd-last-release
# to grab latest patch release; bump this up for every minor release
UPGRADE_VER=$(git tag -l --sort=-version:refname "v3.3.*" | head -1)
if [ -n "$MANUAL_VER" ]; then
# in case, we need to test against different version
UPGRADE_VER=$MANUAL_VER
fi
if [[ -z ${UPGRADE_VER} ]]; then
UPGRADE_VER="v3.3.0"
log_warning "fallback to" ${UPGRADE_VER}
fi
local file="etcd-$UPGRADE_VER-linux-$GOARCH.tar.gz"
log_callout "Downloading $file"
set +e
curl --fail -L "https://github.com/etcd-io/etcd/releases/download/$UPGRADE_VER/$file" -o "/tmp/$file"
local result=$?
set -e
case $result in
0) ;;
*) log_error "--- FAIL:" ${result}
return $result
;;
esac
tar xzvf "/tmp/$file" -C /tmp/ --strip-components=1
mkdir -p ./bin
mv /tmp/etcd ./bin/etcd-last-release
}
function mod_tidy_for_module {
# Watch for upstream solution: https://github.com/golang/go/issues/27005
local tmpModDir
tmpModDir=$(mktemp -d --suffix "etcd-mod")
run cp "./go.mod" "./go.sum" "${tmpModDir}" || return 2
# Guarantees keeping go.sum minimal
# If this is causing too much problems, we should
# stop controlling go.sum at all.
rm go.sum
run go mod tidy || return 2
set +e
local tmpFileGoModInSync
diff -C 5 "${tmpModDir}/go.mod" "./go.mod"
tmpFileGoModInSync="$?"
local tmpFileGoSumInSync
diff -C 5 "${tmpModDir}/go.sum" "./go.sum"
tmpFileGoSumInSync="$?"
set -e
# Bring back initial state
mv "${tmpModDir}/go.mod" "./go.mod"
mv "${tmpModDir}/go.sum" "./go.sum"
if [ "${tmpFileGoModInSync}" -ne 0 ]; then
log_error "${PWD}/go.mod is not in sync with 'go mod tidy'"
return 255
fi
if [ "${tmpFileGoSumInSync}" -ne 0 ]; then
log_error "${PWD}/go.sum is not in sync with 'rm go.sum; go mod tidy'"
return 255
fi
}
function mod_tidy_pass {
run_for_modules mod_tidy_for_module
}
########### MAIN ###############################################################
function run_pass {
local pass="${1}"
shift 1
log_callout -e "\n'${pass}' started at $(date)"
if "${pass}_pass" "$@" ; then
log_success "'${pass}' completed at $(date)"
else
log_error "FAIL: '${pass}' failed at $(date)"
exit 255
fi
}
for pass in $PASSES; do
run_pass "${pass}" "${@}"
done
log_success "SUCCESS"