forked from oracle/weblogic-kubernetes-operator
-
Notifications
You must be signed in to change notification settings - Fork 0
/
run.sh
executable file
·3399 lines (2766 loc) · 121 KB
/
run.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/bin/bash
# Copyright 2017, 2018, Oracle Corporation and/or its affiliates. All rights reserved.
# Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl.
# ---------------------------------------------------------
# Oracle WebLogic Kubernetes Operator Acceptance Test Suite
# ---------------------------------------------------------
#
# -----------------
# Summary and Usage
# -----------------
#
# This script builds the operator, runs a series of acceptance tests,
# and archives the results into tar.gz files upon completion.
#
# It currently runs in three modes, "Wercker", "Jenkins",
# and "standalone" Oracle Linux, where the mode is controlled by
# the WERCKER and JENKINS environment variables described below.
# The default is "standalone".
#
# Steps to run this script:
#
# (1) set the optional env vars described below
# (2) call "cleanup.sh" (more on cleanup below)
# (3) call "run.sh" without any parameters
#
# A succesfull run will have a trace statement with the
# words 'Acceptance Test Suite Completed' and have
# an exit status of 0.
#
# A failed run will have a trace statement with the
# word 'FAIL', and a non-zero exit status.
#
# -------------------------
# Test Cleanup (cleanup.sh)
# -------------------------
#
# To cleanup before or after a run, set the RESULT_ROOT and PV_ROOT
# env vars that are described below to the same values that were
# used when calling run.sh, and then call "cleanup.sh".
#
# Cleanup deletes test temporary files, test PV directories, and
# test kubernetes artifacts (such as pods, services, secrets, etc).
# It will not delete the aforementioned tar.gz archive files
# which run.sh creates after a run completes or fails.
#
# -----------------------
# Logging Levels & Output
# -----------------------
#
# The test has different levels of logging verboseness. Output that begins with:
#
# - ##TEST_INFO - is very concise (one line per test (either PASS or FAIL)).
# - [timestamp] - is concise.
# - + - is verbose.
# - Anything else - is semi-verbose.
#
# By default stand-alone mode copies all output (verbose included)
# to /tmp/test_suite.out, and echos non-verbose output to stdout. Jenkins
# and Wercker modes similarly only echo non-verbose output to stdout by
# default, but they do not copy output to /tmp/test_suite.out.
#
# To echo verbose output to stdout, set VERBOSE to true (see VERBOSE
# env var setting below).
#
# ------------------------
# Test Settings (Env Vars)
# ------------------------
#
# This script accepts optional env var overrides:
#
# RESULT_ROOT The root directory to use for the tests temporary files.
# See "Directory Configuration and Structure" below for
# defaults and a detailed description of test directories.
#
# PV_ROOT The root directory on the kubernetes cluster
# used for persistent volumes.
# See "Directory Configuration and Structure" below for
# defaults and a detailed description of test directories.
#
# LB_TYPE Load balancer type. Can be 'TRAEFIK', 'VOYAGER', or 'APACHE'.
# Default is 'TRAEFIK'.
#
# VERBOSE Set to 'true' to echo verbose tracing to stdout.
# Default is 'false'.
#
# DEBUG_OUT Set to 'tee' to echo various command output to stdout that
# is normally only stored in files. Set to 'cat' to echo such
# commands via 'cat' instead of 'tee'. Set to 'file' to
# only put the output in files. Default is 'file'.
#
# QUICKTEST When set to "true", limits testing to a subset of
# of the tests.
#
# WERCKER Set to true if invoking from Wercker, set
# to false or "" if running stand-alone or from Jenkins.
# Default is "".
#
# JENKINS Set to true if invoking from Jenkins, set
# to false or "" if running stand-alone or from Wercker.
# Default is "".
#
# NODEPORT_HOST DNS name of a Kubernetes worker node.
# Default is the local host's hostname.
#
# JVM_ARGS JVM_ARGS to pass to WebLogic Servers.
# Default is "-Dweblogic.StdoutDebugEnabled=false".
#
# BRANCH_NAME Git branch name.
# Default is determined by calling 'git branch'.
#
# LEASE_ID Set to a unique value to (A) periodically renew a lease on
# the k8s cluster that indicates that no other
# run.sh should attempt to use the cluster, and (B)
# delete this lease when run.sh completes.
# If "true" the caller must previously
# obtain the lease external to the run.sh
# using "lease.sh -o $LEASE_ID", where ID corresponds
# to a process that's expected to continue
# throughout the run (typically the parent
# process) or some uuid potentially generated by uuidgen
# command or some-such..
#
# The following additional overrides are currently only used when
# WERCKER=true:
#
# IMAGE_TAG_OPERATOR Docker image tag for operator.
# Default generated based off the BRANCH_NAME.
#
# IMAGE_NAME_OPERATOR Docker image name for operator.
# Default is wlsldi-v2.docker.oraclecorp.com/weblogic-operator
#
# IMAGE_PULL_POLICY_OPERATOR Default 'Never'.
# IMAGE_PULL_SECRET_OPERATOR Default ''.
# IMAGE_PULL_SECRET_NAME_WEBLOGIC Default ''.
#
# -------------------------------------
# Directory configuration and structure
# -------------------------------------
#
# Main external env vars:
#
# RESULT_ROOT Root path for local test files.
# PV_ROOT Root NFS path behind PV/C directories. This must have permissions
# suitable for WL pods to add files (default UID 1000 group ???)
#
# Defaults for RESULT_ROOT & PV_ROOT:
#
# Test Mode RESULT_ROOT PV_ROOT Where Initialized
# ----------- ---------------------------------- ------- ------------------
# stand-alone /scratch/$USER/wl_k8s_test_results <--same run.sh/cleanup.sh defaults
# Jenkins /scratch/k8s_dir <--same deploy.sh - which calls run.sh
# Wercker /tmp/inttest /scratch wercker.yml - which calls run.sh/cleanup.sh
#
# 'Physical' subdirectories created by test:
#
# Local tmp files: RESULT_ROOT/acceptance_test_tmp/...
#
# PV dirs K8S NFS: PV_ROOT/acceptance_test_pv/domain-storage/...
#
# Archives of above: PV_ROOT/acceptance_test_pv_archive/...
# RESULT_ROOT/acceptance_test_tmp_archive/...
#
# 'Logical' to 'Physical' K8S PV/PVC mappings:
#
# 'Logical' 'Actual'
# job.sh job: /scratch <--> PV_ROOT on K8S machines
# domain pod: /shared <--> PV_ROOT/acceptance_test_pv/persistentVolume-${domain_uid} on K8S machines
#
# ----------------------------------------------
# Instructions for adding a new acceptance test:
# ----------------------------------------------
#
# * Must define as a function with a name prefix of "test_".
# * Must be called from the test_suite function.
# * Must not call another test.
#
# * Mark the entry and exit of each test as follows:
#
# - First call in test must be
# 'declare_new_test <version> <optional description - no spaces>*'
#
# - The test function name and its declare_new_test args must together form
# a unique string that stays the same between runs
# (typically 'declare_new_test 1 "$@"' is sufficient).
#
# - Last call in test must be either 'declare_test_pass' or 'fail' (more on 'fail' below).
#
# * Fail tests using the 'fail' function. Calling fail fails the current test and
# exits the entire acceptance test. fail can be called from nested functions.
#
# * Use the 'trace' function for less verbose tracing.
#
# * For verbose tracing, do not use trace (use echo or some-such) and
# prepend each line with a "+". (The '+' helps down-stream callers filter
# out verbose tracing.)
#
# * Helper functions that check state should be named starting with "verify_"
# or "confirm_" and should call 'fail' on a failure.
#
# * If a test needs a new operator or domain, add it via a 'op_define'
# or 'dom_define' call at the beginning of the test_suite fn, reference
# it using its 'key', and retrieve its values using 'op_get' or 'dom_get'.
#
# Test utilities
function processJson {
python -c "
import sys, json
j=json.load(sys.stdin)
$1
"
}
# Attempt to renew the k8s lease if $LEASE_ID is set. This should be called
# every few minutes throughout the run, and is currently called at the start
# of each test. See LEASE_ID instructions above for instructions about
# how to obtain the lease prior to calling run.sh.
function renewLease {
if [ ! "$LEASE_ID" = "" ]; then
# RESULT_DIR may not have been created yet, so use /tmp
local outfile=/tmp/acc_test_renew_lease.out
$SCRIPTPATH/lease.sh -r "$LEASE_ID" 2>&1 | opt_tee $outfile
if [ $? -ne 0 ]; then
trace "Lease renew error:"
echo "" >> $outfile
echo "ERROR: Could not renew lease on k8s cluster for LEASE_ID=\"$LEASE_ID\"." >> $outfile
echo "Used '$SCRIPTPATH/lease.sh -r \"$LEASE_ID\"' to try renew the lease." >> $outfile
echo "Some of the potential reasons for this failure are that another run" >> $outfile
echo "may have obtained the lease, the lease may have been externally " >> $outfile
echo "deleted, or the caller of run.sh may have forgotten to obtain the" >> $outfile
echo "lease before calling run.sh (using 'lease.sh -o \"$LEASE_ID\"'). " >> $outfile
echo "To force delete a lease no matter who owns the lease," >> $outfile
echo "call 'lease.sh -f' or 'kubernetes delete cm acceptance-test-lease'" >> $outfile
echo "(this should only be done when sure there's no current run.sh " >> $outfile
echo "that owns the lease). To view the current lease holder," >> $outfile
echo "use 'lease.sh -s'. To disable this lease check, do not set" >> $outfile
echo "the LEASE_ID environment variable. For more information, see" >> $outfile
echo "LEASE_ID in the instructions embedded at the top of run.sh." >> $outfile
echo "" >> $outfile
cat $outfile
fail "Could not renew lease on k8s cluster"
fi
rm -f $outfile
trace "Renewed lease."
fi
}
# Test tracing and failure handling
#
# declare_new_test - call this at beginning of each test fn
# declare_new_test_from_trap - call this at ctrl-c interrupt trap
# declare_test_pass - call at the end of test that passes
# trace - trace
# fail - fail a test and exit
#
# NOTES: The declare_new_test calls must generate a unique TEST_ID
# that stays the same between runs. If the same
# test is run twice, it must use different params
# in order to generate a different description.
#
# Calling declare_new_test before a previous declare passes or fails
# generates a fail.
#
# Related:
# declare_test_fail - report test failed, called by 'fail'
# state_dump - dump current state to files
# called by 'fail' and at end of the run
# archive.sh - tar.gz all results, called by state_dump
# ctrl_c - traps users ctrl-c interrupt and calls fail
#
#
# declare_reset
# - implicitly called from a declare_test_pass or declare_test_fail
# - must also call at start of the run
# - resets TEST_ID and TEST_VERSION to a well known NULL value
#
function declare_reset {
export TEST_VERSION="NULL"
export TEST_ID="NULL"
}
#
# declare_test_trace PASS|FAIL|START [echo]
# - internal helper
# - generates trace with contents ##TEST_INFO:${GIT_ABBREVIATED_COMMIT_HASH}:${TEST_ID}:${TEST_VERSION}:${1}##
# - also echos same if 'echo' passed as the second parameter
#
function declare_test_trace {
local GIT_ABBREVIATED_COMMIT_HASH="${WERCKER_GIT_COMMIT:-`git log --pretty=format:%h -n 1`}"
local str="##TEST_INFO:${GIT_ABBREVIATED_COMMIT_HASH}:${TEST_ID}:${TEST_VERSION}:${1}##"
trace "$str"
if [ "$2" = echo ]; then
echo "$str"
fi
}
#
# declare_new_test <version> [<testargs...>]
# - must call at beginning of each test
# - sets TEST_VERSION according to $1
# - sets TEST_ID based on caller's fn name and <testargs...>
# - pairs with a subsequent call to 'declare_test_pass' or 'fail'
# - fails if previous test did not call declare_test_pass or fail
#
function declare_new_test {
if [ "$#" -eq 0 ] ; then
fail "Missing argument(s). Expected <version> [<testargs...>]"
fi
if [ ! "$TEST_VERSION" = "NULL" ]; then
fail "New test declared before previous test failed. Previous TEST_ID=$TEST_ID"
fi
export TEST_VERSION="${1}"
export TEST_ID="${FUNCNAME[1]}"
shift
while [ ! "$1" = "" ]; do
export TEST_ID="${TEST_ID}.${1}"
shift
done
declare_test_trace START
renewLease
}
#
# declare_new_test_from_trap <version> <testname>
# - call only from a trap that's about to call fail
# - the trap obliterates exports like TEST_VERSION and TEST_ID, so we need to set them to something logical.
#
function declare_new_test_from_trap {
export TEST_VERSION="${1?}"
export TEST_ID="${2?}"
}
# declare_test_fail
# - do not call this directly from a test
# - implicitly called from fail
# - pairs with an earlier call to declare_new_test
#
function declare_test_fail {
# the primary reason for this method is to echo the test failure in a special
# format: ##TEST_INFO:commitID:testID:testVersion:FAIL##
declare_test_trace FAIL echo
declare_reset
}
#
# declare_test_pass
# - must call at the end of each test
# - pairs with an earlier call to declare_new_test
#
function declare_test_pass {
# the primary reason for this method is to echo the test pass in a special
# format: ##TEST_INFO:commitID:testID:testVersion:PASS##
if [ "$TEST_VERSION" = "NULL" -o "$TEST_VERSION" = "" ]; then
fail "No current test."
fi
declare_test_trace PASS echo
declare_reset
}
#
# trace <message>
#
function trace {
#Date reported in same format as oper-log for easier correlation. 01-22-2018T21:49:01
#See also a similar echo in function fail
echo "[`date '+%m-%d-%YT%H:%M:%S'`] [secs=$SECONDS] [test=$TEST_ID] [fn=${FUNCNAME[1]}]: ""$@"
}
#
# opt_tee <filename>
# Use this function in place of 'tee' for most use cases.
# See the description of 'DEBUG_OUT' above for information about this function
#
function opt_tee {
local filename="${1?}"
if [ "$DEBUG_OUT" = "tee" ]; then
tee $filename
elif [ "$DEBUG_OUT" = "cat" ]; then
cat > $filename
cat $filename
else
cat > $filename
fi
echo "[`date '+%m-%d-%YT%H:%M:%S'`] [secs=$SECONDS] [test=$TEST_ID] [fn=${FUNCNAME[1]}]: Output from last command is in ""$1"
}
#
# state_dump <dir-suffix>
# - called at the end of a run, and from fail
# - places k8s logs, descriptions, etc in directory $RESULT_DIR/state-dump-$1
# - calls archive.sh on RESULT_DIR locally, and on PV_ROOT via a job
# - IMPORTANT: this method must never call fail (since it is called from fail)
# - IMPORTANT: this method should not rely on exports
#
function state_dump {
if [ -z "$RESULT_DIR" ]; then
# exports can apparently be lost when the trap catches a ^C
trace Exports have been lost. Trying to recreate.
local RESULT_DIR="`cat /tmp/test_suite.result_root`/acceptance_test_tmp"
local PV_ROOT="`cat /tmp/test_suite.pv_root`"
local PROJECT_ROOT="`cat /tmp/test_suite.project_root`"
local SCRIPTPATH="$PROJECT_ROOT/src/integration-tests/bash"
local LEASE_ID="`cat /tmp/test_suite.lease_id`"
local DEBUG_OUT="`cat /tmp/test_suite.debug_out`"
if [ ! -d "$RESULT_DIR" ]; then
trace State dump exiting early. RESULT_DIR \"$RESULT_DIR\" does not exist or is not a directory.
return
fi
fi
local DUMP_DIR=$RESULT_DIR/state-dump-${1:?}
trace Starting state dump. Dumping state to directory ${DUMP_DIR}
mkdir -p ${DUMP_DIR}
# Test output is captured to ${TESTOUT} when run.sh is run stand-alone
if [ -f ${TESTOUT:-NoSuchFile.out} ]; then
trace Copying ${TESTOUT} to ${DUMP_DIR}/test_suite.out
cp ${TESTOUT} ${DUMP_DIR}/test_suite.out
fi
# dumping kubectl state
# get domains is in its own command since this can fail if domain CRD undefined
trace Dumping kubectl gets to kgetmany.out and kgetdomains.out in ${DUMP_DIR}
kubectl get all,crd,cm,pv,pvc,ns,roles,rolebindings,clusterroles,clusterrolebindings,secrets --show-labels=true --all-namespaces=true 2>&1 | opt_tee ${DUMP_DIR}/kgetmany.out
kubectl get domains --show-labels=true --all-namespaces=true 2>&1 | opt_tee ${DUMP_DIR}/kgetdomains.out
# Get all pod logs and redirect/copy to files
set +x
local namespaces="`kubectl get namespaces | egrep -v -e "(STATUS|kube)" | awk '{ print $1 }'`"
set -x
local namespace
trace "Copying logs and describes to pod-log.NAMESPACE.PODNAME and pod-describe.NAMESPACE.PODNAME in ${DUMP_DIR}"
for namespace in $namespaces; do
set +x
local pods="`kubectl get pods -n $namespace --ignore-not-found | egrep -v -e "(STATUS)" | awk '{print $1}'`"
set -x
local pod
for pod in $pods; do
local logfile=${DUMP_DIR}/pod-log.${namespace}.${pod}
local descfile=${DUMP_DIR}/pod-describe.${namespace}.${pod}
kubectl log $pod -n $namespace 2>&1 | opt_tee $logfile
kubectl describe pod $pod -n $namespace 2>&1 | opt_tee $descfile
done
done
# use a job to archive PV, /scratch mounts to PV_ROOT in the K8S cluster
trace "Archiving pv directory using a kubernetes job. Look for it on k8s cluster in $PV_ROOT/acceptance_test_pv_archive"
local outfile=${DUMP_DIR}/archive_pv_job.out
$SCRIPTPATH/job.sh "/scripts/archive.sh /scratch/acceptance_test_pv /scratch/acceptance_test_pv_archive" 2>&1 | opt_tee ${outfile}
if [ "$?" = "0" ]; then
trace Job complete.
else
trace Job failed. See ${outfile}.
fi
if [ ! "$LEASE_ID" = "" ]; then
# release the lease if we own it
${SCRIPTPATH}/lease.sh -d "$LEASE_ID" 2>&1 | opt_tee ${RESULT_DIR}/release_lease.out
if [ "$?" = "0" ]; then
trace Lease released.
else
trace Lease could not be released:
cat /${RESULT_DIR}/release_lease.out
fi
fi
# now archive all the local test files
$SCRIPTPATH/archive.sh "${RESULT_DIR}" "${RESULT_DIR}_archive"
trace Done with state dump
}
#
# fail <message>
# - report the message, fail the run, and exit the script
# - calls state_dump (which calls archive)
#
function fail {
set +x
#See also a similar echo in function trace
echo "[`date '+%m-%d-%YT%H:%M:%S'`] [secs=$SECONDS] [test=$TEST_ID] [fn=${FUNCNAME[1]}]: [FAIL] ""$@"
#echo current test failure in a special format
declare_test_fail
echo "Stack trace:"
local deptn=${#FUNCNAME[@]}
local i
for ((i=1; i<$deptn; i++)); do
local func="${FUNCNAME[$i]}"
local line="${BASH_LINENO[$((i-1))]}"
local src="${BASH_SOURCE[$((i-1))]}"
printf '%*s' $i '' # indent
echo "at: $func(), $src, line $line"
done
state_dump fail
echo "[`date '+%m-%d-%YT%H:%M:%S'`] [secs=$SECONDS] [test=$TEST_ID] [fn=${FUNCNAME[1]}]: [FAIL] Exiting with status 1"
exit 1
}
trap ctrl_c INT
function ctrl_c() {
declare_new_test_from_trap 1 run_aborted_with_ctrl_c
# disable the trap:
trap - INT
set -o pipefail
fail "Trapped CTRL-C"
}
function clean_jenkins {
trace Cleaning.
/usr/local/packages/aime/ias/run_as_root "${SCRIPTPATH}/clean_docker_k8s.sh -y"
}
function setup_jenkins {
trace Setting up.
/usr/local/packages/aime/ias/run_as_root "sh ${SCRIPTPATH}/install_docker_k8s.sh -y -u wls -v ${K8S_VERSION}"
set +x
. ~/.dockerk8senv
set -x
id
trace "Pull and tag the images we need"
docker login -u teamsldi_us@oracle.com -p $docker_pass wlsldi-v2.docker.oraclecorp.com
docker images
docker pull wlsldi-v2.docker.oraclecorp.com/store-weblogic-12.2.1.3:latest
docker tag wlsldi-v2.docker.oraclecorp.com/store-weblogic-12.2.1.3:latest store/oracle/weblogic:12.2.1.3
docker pull wlsldi-v2.docker.oraclecorp.com/store-serverjre-8:latest
docker tag wlsldi-v2.docker.oraclecorp.com/store-serverjre-8:latest store/oracle/serverjre:8
docker pull wlsldi-v2.docker.oraclecorp.com/weblogic-webtier-apache-12.2.1.3.0:latest
docker tag wlsldi-v2.docker.oraclecorp.com/weblogic-webtier-apache-12.2.1.3.0:latest store/oracle/apache:12.2.1.3
# create a docker image for the operator code being tested
export JAR_VERSION="`grep -m1 "<version>" pom.xml | cut -f2 -d">" | cut -f1 -d "<"`"
trace "Running docker build --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy --build-arg no_proxy=$no_proxy -t "${IMAGE_NAME_OPERATOR}:${IMAGE_TAG_OPERATOR}" --build-arg VERSION=$JAR_VERSION --no-cache=true ."
docker build --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy --build-arg no_proxy=$no_proxy -t "${IMAGE_NAME_OPERATOR}:${IMAGE_TAG_OPERATOR}" --build-arg VERSION=$JAR_VERSION --no-cache=true .
docker images
trace "Helm installation starts"
wget -q -O /tmp/helm-v2.8.2-linux-amd64.tar.gz https://kubernetes-helm.storage.googleapis.com/helm-v2.8.2-linux-amd64.tar.gz
mkdir /tmp/helm
tar xzf /tmp/helm-v2.8.2-linux-amd64.tar.gz -C /tmp/helm
chmod +x /tmp/helm/linux-amd64/helm
/usr/local/packages/aime/ias/run_as_root "cp /tmp/helm/linux-amd64/helm /usr/bin/"
rm -rf /tmp/helm
helm init
trace "Helm is configured."
# we always use HELM for the operator now
if [ ! -x "$(command -v helm)" ]; then
fail "We always use HELM charts for the operator now. But helm binary not found in path. the helm installation in this function must have failed "
fi
}
# setup_local is for arbitrary dev hosted linux - it assumes docker & k8s are already installed
function setup_local {
trace "Pull and tag the images we need"
docker pull wlsldi-v2.docker.oraclecorp.com/store-weblogic-12.2.1.3:latest
docker tag wlsldi-v2.docker.oraclecorp.com/store-weblogic-12.2.1.3:latest store/oracle/weblogic:12.2.1.3
docker pull wlsldi-v2.docker.oraclecorp.com/store-serverjre-8:latest
docker tag wlsldi-v2.docker.oraclecorp.com/store-serverjre-8:latest store/oracle/serverjre:8
docker pull wlsldi-v2.docker.oraclecorp.com/weblogic-webtier-apache-12.2.1.3.0:latest
docker tag wlsldi-v2.docker.oraclecorp.com/weblogic-webtier-apache-12.2.1.3.0:latest store/oracle/apache:12.2.1.3
if [ ! -x "$(command -v helm)" ]; then
fail "We always use HELM charts for the operator now. But helm binary not found in path, helm must be pre-installed prior to running integration tests locally"
fi
}
function setup_wercker {
trace "Perform setup for running in wercker"
trace "Install tiller"
kubectl create serviceaccount --namespace kube-system tiller
kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller
# Note: helm init --wait would wait until tiller is ready, and requires helm 2.8.2 or above
helm init --service-account=tiller --wait
helm version
kubectl get po -n kube-system
trace "Existing helm charts "
helm ls
trace "Deleting installed helm charts"
helm list --short --all | xargs -L1 helm delete --purge
trace "After helm delete, list of installed helm charts is: "
helm ls --all
if [ ! -x "$(command -v helm)" ]; then
fail "We always use HELM charts for the operator now. But helm binary not found in path. the helm installation in this function must have failed "
fi
trace "Completed setup_wercker"
}
function create_image_pull_secret_jenkins {
trace "Creating Secret"
kubectl create secret docker-registry wlsldi-secret \
--docker-server=wlsldi-v2.docker.oraclecorp.com \
--docker-username=teamsldi_us@oracle.com \
--docker-password=$docker_pass \
--docker-email=teamsldi_us@oracle.com 2>&1 | sed 's/^/+' 2>&1
trace "Checking Secret"
local SECRET="`kubectl get secret wlsldi-secret | grep wlsldi | wc | awk ' { print $1; }'`"
if [ "$SECRET" != "1" ]; then
fail 'secret wlsldi-secret was not created successfully'
fi
}
function create_image_pull_secret_wercker {
local namespace=${1:-default}
trace "Creating Docker Secret"
kubectl create secret docker-registry $IMAGE_PULL_SECRET_WEBLOGIC \
--docker-server='$REPO_SERVER' \
--docker-username='$REPO_USERNAME' \
--docker-password='$REPO_PASSWORD' \
--docker-email='$REPO_EMAIL' \
-n $namespace 2>&1 | sed 's/^/+' 2>&1
trace "Checking Secret"
local SECRET="`kubectl get secret $IMAGE_PULL_SECRET_WEBLOGIC | grep $IMAGE_PULL_SECRET_WEBLOGIC | wc | awk ' { print $1; }'`"
if [ "$SECRET" != "1" ]; then
fail 'secret $IMAGE_PULL_SECRET_WEBLOGIC was not created successfully'
fi
trace "Creating Registry Secret"
kubectl create secret docker-registry $IMAGE_PULL_SECRET_OPERATOR \
--docker-server='$REPO_SERVER' \
--docker-username='$REPO_USERNAME' \
--docker-password='$REPO_PASSWORD' \
--docker-email='$REPO_EMAIL' \
-n $namespace 2>&1 | sed 's/^/+' 2>&1
trace "Checking Secret"
local SECRET="`kubectl get secret $IMAGE_PULL_SECRET_OPERATOR | grep $IMAGE_PULL_SECRET_OPERATOR | wc | awk ' { print $1; }'`"
if [ "$SECRET" != "1" ]; then
fail 'secret $IMAGE_PULL_SECRET_OPERATOR was not created successfully'
fi
}
# op_define OP_KEY NAMESPACE TARGET_NAMESPACES EXTERNAL_REST_HTTPSPORT
# sets up table of operator values.
#
# op_get OP_KEY
# gets an operator value
#
# op_echo OP_KEY
# lists the values
#
# Usage example:
# op_define myop mynamespace ns1,ns2 34007
# opkey="myop"
# echo Defined operator $opkey with `op_echo $opkey`
# local nspace="`op_get $opkey NAMESPACE`"
#
function op_define {
if [ "$#" != 4 ] ; then
fail "requires 4 parameters: OP_KEY NAMESPACE TARGET_NAMESPACES EXTERNAL_REST_HTTPSPORT"
fi
local opkey="`echo \"${1?}\" | sed 's/-/_/g'`"
eval export OP_${opkey}_NAMESPACE="$2"
eval export OP_${opkey}_TARGET_NAMESPACES="$3"
eval export OP_${opkey}_EXTERNAL_REST_HTTPSPORT="$4"
# generated TMP_DIR for operator = $USER_PROJECTS_DIR/weblogic-operators/$NAMESPACE :
eval export OP_${opkey}_TMP_DIR="$USER_PROJECTS_DIR/weblogic-operators/$2"
#verbose tracing starts with a +
op_echo_all $1 | sed 's/^/+/'
trace Defined operator ${1}.
}
function op_get {
local opkey="`echo \"${1?}\" | sed 's/-/_/g'`"
eval "echo \${OP_${opkey}_${2?}}"
}
function op_echo_all {
local opkey="`echo \"${1?}\" | sed 's/-/_/g'`"
env | grep "^OP_${opkey}_"
}
function operator_ready_wait {
if [ "$#" != 1 ] ; then
fail "requires 1 parameter: operatorKey"
fi
local OP_KEY=${1}
local OPERATOR_NS="`op_get $OP_KEY NAMESPACE`"
local TMP_DIR="`op_get $OP_KEY TMP_DIR`"
local namespace=$OPERATOR_NS
trace Waiting for operator deployment to be ready...
local AVAILABLE="0"
local max=30
local count=0
while [ "$AVAILABLE" != "1" -a $count -lt $max ] ; do
sleep 10
local AVAILABLE=`kubectl get deploy weblogic-operator -n $namespace -o jsonpath='{.status.availableReplicas}'`
trace "status is $AVAILABLE, iteration $count of $max"
local count=`expr $count + 1`
done
if [ "$AVAILABLE" != "1" ]; then
kubectl get deploy weblogic-operator -n ${namespace}
kubectl describe deploy weblogic-operator -n ${namespace}
kubectl describe pods -n ${namespace}
fail "The WebLogic operator deployment is not available, after waiting 300 seconds"
fi
}
function deploy_operator {
if [ "$#" != 1 ] ; then
fail "requires 1 parameter: opkey"
fi
local opkey=${1}
local NAMESPACE="`op_get $opkey NAMESPACE`"
local TARGET_NAMESPACES="`op_get $opkey TARGET_NAMESPACES`"
local EXTERNAL_REST_HTTPSPORT="`op_get $opkey EXTERNAL_REST_HTTPSPORT`"
local TMP_DIR="`op_get $opkey TMP_DIR`"
if [ "$WERCKER" = "true" ]; then
create_image_pull_secret_wercker $NAMESPACE
fi
trace 'customize the yaml'
mkdir -p $TMP_DIR
local inputs="$TMP_DIR/weblogic-operator-values.yaml"
# generate certificates
$PROJECT_ROOT/kubernetes/samples/scripts/generate-external-rest-identity.sh DNS:${NODEPORT_HOST} > $inputs
trace 'customize the inputs yaml file to add test namespace'
echo "domainNamespaces:" >> $inputs
for i in $(echo $TARGET_NAMESPACES | sed "s/,/ /g")
do
echo " - $i" >> $inputs
done
echo "imagPullPolicy: ${IMAGE_PULL_POLICY_OPERATOR}" >> $inputs
echo "image: ${IMAGE_NAME_OPERATOR}:${IMAGE_TAG_OPERATOR}" >> $inputs
echo "imagePullSecrets: { name: ${IMAGE_PULL_SECRET_OPERATOR} }" >> $inputs
echo "externalRestEnabled: true" >> $inputs
trace 'customize the inputs yaml file to set the java logging level to $LOGLEVEL_OPERATOR'
echo "javaLoggingLevel: \"$LOGLEVEL_OPERATOR\"" >> $inputs
echo "externalRestHttpsPort: ${EXTERNAL_REST_HTTPSPORT}" >> $inputs
echo "serviceAccount: weblogic-operator" >> $inputs
trace "Contents after customization in file $inputs"
cat $inputs
local outfile="${TMP_DIR}/create-weblogic-operator-helm.out"
trace "Run helm install to deploy the weblogic operator, see \"$outfile\" for tracking."
cd $PROJECT_ROOT/kubernetes/charts
helm install weblogic-operator --name ${opkey} --namespace ${NAMESPACE} -f $inputs 2>&1 | opt_tee ${outfile}
trace "helm install output:"
cat $outfile
operator_ready_wait $opkey
if [ "$?" = "0" ]; then
# Prepend "+" to detailed debugging to make it easy to filter out
cat ${outfile} | sed 's/^/+/g'
trace Script complete.
else
cat ${outfile}
fail Script failed.
fi
# Prepend "+" to detailed debugging to make it easy to filter out
echo 'weblogic-operator.yaml contents:' 2>&1 | sed 's/^/+/' 2>&1
cat $TMP_DIR/weblogic-operator.yaml 2>&1 | sed 's/^/+/' 2>&1
echo 2>&1 | sed 's/^/+/' 2>&1
trace "Checking the operator pods"
#TODO It looks like this code isn't checking if REPLICA_SET, POD_TEMPLATE, and POD have expected values, is that intentional?
local namespace=$NAMESPACE
local REPLICA_SET=`kubectl describe deploy weblogic-operator -n ${namespace} | grep NewReplicaSet: | awk ' { print $2; }'`
local POD_TEMPLATE=`kubectl describe rs ${REPLICA_SET} -n ${namespace} | grep ^Name: | awk ' { print $2; } '`
local POD=`kubectl get pods -n ${namespace} | grep $POD_TEMPLATE | awk ' { print $1; } '`
trace "Checking image for pod $POD"
local IMAGE=`kubectl describe pod $POD -n ${namespace} | grep "Image:" | awk ' { print $2; } '`
if [ "$IMAGE" != "${IMAGE_NAME_OPERATOR}:${IMAGE_TAG_OPERATOR}" ]; then
fail "pod image should be ((${IMAGE_NAME_OPERATOR}:${IMAGE_TAG_OPERATOR})) but image is ((${IMAGE}))"
fi
}
function test_first_operator {
declare_new_test 1 "$@"
if [ "$#" != 1 ] ; then
fail "requires 1 parameter: opkey"
fi
local OP_KEY=${1}
deploy_operator $OP_KEY
verify_no_domain_via_oper_rest $OP_KEY
declare_test_pass
}
# Create another operator in a new namespace named after the operator, managing a new namespace
# the specified existing domain should not be affacted
function test_second_operator {
declare_new_test 1 "$@"
if [ "$#" != 2 ] ; then
fail "requires 2 parameters: new_opkey dom_key_for_existing_domain"
fi
local OP_KEY="${1}"
local DOM_KEY="${2}"
deploy_operator $OP_KEY
verify_domain $DOM_KEY
declare_test_pass
}
# dom_define DOM_KEY OP_KEY NAMESPACE DOMAIN_UID STARTUP_CONTROL WL_CLUSTER_NAME WL_CLUSTER_TYPE MS_BASE_NAME ADMIN_PORT ADMIN_WLST_PORT ADMIN_NODE_PORT MS_PORT LOAD_BALANCER_WEB_PORT LOAD_BALANCER_DASHBOARD_PORT
# Sets up a table of domain values: all of the above, plus TMP_DIR which is derived.
#
# dom_get DOM_KEY <value>
# Gets a domain value.
#
# dom_echo_all DOM_KEY
# Lists domain values for the given key.
#
# Usage example:
# dom_define mydom default domain1 cluster-1 managed-server 7001 30012 30701 8001 30305 30315
# local DOM_KEY=mydom
# local nspace="`dom_get $DOM_KEY NAMESPACE`"
# echo Defined operator $opkey with `dom_echo_all $DOM_KEY`
#
function dom_define {
if [ "$#" != 14 ] ; then
fail "requires 14 parameters: DOM_KEY OP_KEY NAMESPACE DOMAIN_UID STARTUP_CONTROL WL_CLUSTER_NAME WL_CLUSTER_TYPE MS_BASE_NAME ADMIN_PORT ADMIN_WLST_PORT ADMIN_NODE_PORT MS_PORT LOAD_BALANCER_WEB_PORT LOAD_BALANCER_DASHBOARD_PORT"
fi
local DOM_KEY="`echo \"${1}\" | sed 's/-/_/g'`"
eval export DOM_${DOM_KEY}_OP_KEY="$2"
eval export DOM_${DOM_KEY}_NAMESPACE="$3"
eval export DOM_${DOM_KEY}_DOMAIN_UID="$4"
eval export DOM_${DOM_KEY}_STARTUP_CONTROL="$5"
eval export DOM_${DOM_KEY}_WL_CLUSTER_NAME="$6"
eval export DOM_${DOM_KEY}_WL_CLUSTER_TYPE="$7"
if [ "$CONFIGURED_CLUSTER_ONLY" = "true" ] && [ "$7" = "DYNAMIC" ]; then
trace "Warning: Forcing domain uid '$4' to be CONFIGURED instead of DYNAMIC, as CONFIGURED_CLUSTER_ONLY=true"
eval export DOM_${DOM_KEY}_WL_CLUSTER_TYPE="CONFIGURED"
fi
eval export DOM_${DOM_KEY}_MS_BASE_NAME="$8"
eval export DOM_${DOM_KEY}_ADMIN_PORT="$9"
eval export DOM_${DOM_KEY}_ADMIN_WLST_PORT="${10}"
eval export DOM_${DOM_KEY}_ADMIN_NODE_PORT="${11}"
eval export DOM_${DOM_KEY}_MS_PORT="${12}"
eval export DOM_${DOM_KEY}_LOAD_BALANCER_WEB_PORT="${13}"
eval export DOM_${DOM_KEY}_LOAD_BALANCER_DASHBOARD_PORT="${14}"
# derive TMP_DIR $USER_PROJECTS_DIR/weblogic-domains/$NAMESPACE-$DOMAIN_UID :
eval export DOM_${DOM_KEY}_TMP_DIR="$USER_PROJECTS_DIR/weblogic-domains/$4"
# we only test loadBalancerExposeAdminPort=true for Apache on domain1
if [ "$LB_TYPE" == "APACHE" ] && [ "$4" == "domain1" ] ; then
export DOM_${DOM_KEY}_LOAD_BALANCER_EXPOSE_ADMIN_PORT="true"
else
export DOM_${DOM_KEY}_LOAD_BALANCER_EXPOSE_ADMIN_PORT="false"
fi
#verbose tracing starts with a +
dom_echo_all $1 | sed 's/^/+/'
trace Defined domain ${1}.
}
function dom_get {
local domkey="`echo \"${1?}\" | sed 's/-/_/g'`"
eval "echo \${DOM_${domkey}_${2?}}"
}
function dom_echo_all {
local domkey="`echo \"${1?}\" | sed 's/-/_/g'`"
env | grep "^DOM_${domkey}_"
}
#
# Usage:
# createVoyagerIngress voyagerIngressYaml namespace domainUID
#
function createVoyagerIngress {
if [ "$#" != 3 ] ; then
fail "requires 1 parameter: voyagerIngressYaml namespace domainUID"
fi
# deploy Voyager Ingress resource
kubectl apply -f $1
local namespace=$2
local domainUID=$3
echo "Checking Voyager Ingress resource..."
local maxwaitsecs=100
local mstart=$(date +%s)
while : ; do
local mnow=$(date +%s)
local vdep=$(kubectl get ingresses.voyager.appscode.com -n ${namespace} | grep ${domainUID}-voyager | wc | awk ' { print $1; } ')
if [ "$vdep" = "1" ]; then
echo "The Voyager Ingress resource ${domainUID}-voyager is created successfully."
break
fi
if [ $((mnow - mstart)) -gt $((maxwaitsecs)) ]; then
fail "The Voyager Ingress resource ${domainUID}-voyager was not created."
fi
sleep 2
done
echo "Wait until HAProxy pod is running..."
local maxwaitsecs=100
local mstart=$(date +%s)
while : ; do
local mnow=$(date +%s)
local st=$(kubectl get pod -n ${namespace} | grep ^voyager-${domainUID}-voyager- | awk ' { print $3; } ')
if [ "$st" = "Running" ]; then
echo "The HAProxy pod for Voyager Ingress ${domainUID}-voyager is running."
break
fi
if [ $((mnow - mstart)) -gt $((maxwaitsecs)) ]; then
fail "The HAProxy pod for Voyager Ingress ${domainUID}-voyager was not created or running."
fi
sleep 5
done
echo "Checking Voyager service..."
local maxwaitsecs=10
local mstart=`date +%s`
while : ; do
local mnow=`date +%s`
local vscv=`kubectl get service ${domainUID}-voyager-stats -n ${namespace} | grep ${domainUID}-voyager-stats | wc | awk ' { print $1; } '`
if [ "$vscv" = "1" ]; then
echo "The service ${domainUID}-voyager-stats is created successfully."
break
fi
if [ $((mnow - mstart)) -gt $((maxwaitsecs)) ]; then
fail "The service ${domainUID}-voyager-stats was not created."
fi
sleep 2
done
echo
}
function create_pv_pvc_non_helm {
if [ "$#" != 1 ] ; then
fail "requires 1 parameter: domainkey"
fi
local NAMESPACE="`dom_get $1 NAMESPACE`"
local DOMAIN_UID="`dom_get $1 DOMAIN_UID`"
local STARTUP_CONTROL="`dom_get $1 STARTUP_CONTROL`"
local WL_CLUSTER_NAME="`dom_get $1 WL_CLUSTER_NAME`"
local WL_CLUSTER_TYPE="`dom_get $1 WL_CLUSTER_TYPE`"
local MS_BASE_NAME="`dom_get $1 MS_BASE_NAME`"
local ADMIN_PORT="`dom_get $1 ADMIN_PORT`"
local ADMIN_WLST_PORT="`dom_get $1 ADMIN_WLST_PORT`"
local ADMIN_NODE_PORT="`dom_get $1 ADMIN_NODE_PORT`"
local MS_PORT="`dom_get $1 MS_PORT`"
if [ -z ${domainUID} ]; then
tmp_dir=${USER_PROJECTS_DIR}/pv-pvcs
else