-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathpulsar-contributor-toolbox-functions.sh
2073 lines (1873 loc) · 67.6 KB
/
pulsar-contributor-toolbox-functions.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# shell functions for working with Pulsar development
# zsh and bash are supported
if [ -z "$PULSAR_CONTRIBUTOR_TOOLBOX" ]; then
if [ -n "$BASH_SOURCE" ]; then
PULSAR_CONTRIBUTOR_TOOLBOX=$(dirname $BASH_SOURCE)
else
# zsh
PULSAR_CONTRIBUTOR_TOOLBOX=${0:a:h}
fi
PULSAR_CONTRIBUTOR_TOOLBOX=$(dirname $PULSAR_CONTRIBUTOR_TOOLBOX)
fi
PTBX_DEFAULT_DOCKER_REPO_PREFIX="lhotari"
PTBX_DEFAULT_DOCKER_IMAGE_PREFIX="pulsar"
PTBX_DEFAULT_JAVA_TEST_IMAGE_NAME="java-test-image"
if [ -f "$HOME/.pulsar_contributor_toolbox" ]; then
source "$HOME/.pulsar_contributor_toolbox"
fi
# alias for refreshing changes
if [ -n "$BASH_SOURCE" ]; then
alias ptbx_refresh="source $BASH_SOURCE"
else
# zsh
alias ptbx_refresh="source ${0:a}"
fi
# add bin directory to path
[[ $(echo $PATH | grep -c "${PULSAR_CONTRIBUTOR_TOOLBOX}/bin") -eq 0 ]] && \
export PATH="$PULSAR_CONTRIBUTOR_TOOLBOX/bin:$PATH"
# runs license checks
function ptbx_run_license_check() {
(
ptbx_cd_git_root
mvn -ntp -DskipTests initialize license:check
)
}
# runs license checks and checkstyle
function ptbx_run_quick_check() {
(
mvn -ntp -T 1C -DskipSourceReleaseAssembly=true -DskipBuildDistribution=true -Dspotbugs.skip=true verify -DskipTests "$@"
)
}
function ptbx_build_coremodules() {
(
ptbx_cd_git_root
local clean_param="clean"
if [[ "$1" == "--noclean" || "$1" == "-nc" ]]; then
clean_param=""
shift
else
ptbx_clean_snapshots
fi
mvn -Pcore-modules,-main -T 1C $clean_param install -DskipTests -Dspotbugs.skip=true -DnarPluginPhase=none "$@"
)
}
function ptbx_build_all() {
(
ptbx_cd_git_root
ptbx_clean_snapshots
command mvn -T 1C clean install -DskipTests -Dspotbugs.skip=true -DShadeTests -DintegrationTests -DBackwardsCompatTests -Dtest=NoneTest -DfailIfNoTests=false "$@"
)
}
function ptbx_build_inttests() {
(
ptbx_cd_git_root
ptbx_clean_snapshots
command mvn -T 1C clean install -DskipTests -Dspotbugs.skip=true -DintegrationTests -Dtest=NoneTest -DfailIfNoTests=false -am -pl tests/integration "$@"
)
}
function ptbx_run_inttest() {
(
ptbx_cd_git_root
export PULSAR_TEST_IMAGE_NAME=apachepulsar/java-test-image:latest
command mvn -T 1C test -DredirectTestOutputToFile=false -DtestRetryCount=0 -Dspotbugs.skip=true -DintegrationTests -pl tests/integration "$@"
)
}
function ptbx_run_systest() {
(
ptbx_cd_git_root
export PULSAR_TEST_IMAGE_NAME=apachepulsar/pulsar-test-latest-version:latest
command mvn -T 1C test -DredirectTestOutputToFile=false -DtestRetryCount=0 -Dspotbugs.skip=true -DintegrationTests -pl tests/integration "$@"
)
}
function ptbx_build_server_distribution() {
(
ptbx_build_server_distribution_full -Pcore-modules,-main "$@"
)
}
function ptbx_server_distribution_license_check() {
(
ptbx_build_server_distribution
ptbx_cd_git_root
src/check-binary-license --no-presto ./distribution/server/target/apache-pulsar-*-bin.tar.gz
)
}
function ptbx_build_server_distribution_full() {
(
ptbx_cd_git_root
ptbx_clean_snapshots
command mvn -T 1C clean install -Dmaven.test.skip=true -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true -pl distribution/server -am "$@"
)
}
function ptbx_server_distribution_license_check_full() {
(
ptbx_build_server_distribution_full
ptbx_cd_git_root
src/check-binary-license ./distribution/server/target/apache-pulsar-*-bin.tar.gz
)
}
function ptbx_clean_snapshots() {
(
if [ -n "$ZSH_NAME" ]; then
setopt nonomatch
fi
ls -d ~/.m2/repository/{org/apache,com/datastax/oss}/pulsar/**/"$(ptbx_project_version)" 2>/dev/null | xargs -r rm -rf
)
}
function ptbx_clean_cppbuild() {
(
ptbx_cd_git_root
cd pulsar-client-cpp
if [ -n "$(find '!' -user $USER)" ]; then
sudo chown -R $USER:$GROUP .
fi
git clean -fdx
)
}
# runs a command until it fails
function ptbx_untilfail() {
(
while $@; do :; done
)
}
# runs a command within docker to limit cpu and memory
function ptbx_docker_run() {
(
local cpus=2
local memory=6g
local platform=""
local no_host_net=0
while [ true ]; do
if [[ "$1" =~ --cpus=.* ]]; then
cpus="${1#*=}"
shift
elif [[ "$1" =~ --memory=.* ]]; then
memory="${1#*=}"
shift
elif [[ "$1" =~ --platform=.* ]]; then
platform="$1"
shift
elif [[ "$1" == --no-host-net ]]; then
no_host_net=1
shift
else
break
fi
done
local host_net_param="--net=host"
if [[ $no_host_net == 1 ]]; then
host_net_param=""
fi
if [[ -z "$platform" ]]; then
if uname -m | grep -q x86_64; then
platform="--platform=linux/amd64"
else
platform="--platform=linux/arm64"
fi
fi
local arch="${platform#*=}"
arch="${arch#*/}"
local testcontainers_param=""
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
additional_groups=()
for gid in $(id -G); do
additional_groups+=("--group-add=$gid")
done
docker run $platform --env-file=<(printenv) --security-opt seccomp=unconfined --cap-add SYS_ADMIN --cpus=$cpus --memory=$memory -u "$UID:${GID:-"$(id -g)"}" "${additional_groups[@]}" $host_net_param -it --rm -v $HOME:$HOME -v /var/run/docker.sock:/var/run/docker.sock -w $PWD -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro ubuntu "$@"
elif [[ "$OSTYPE" == "darwin"* ]]; then
local imagename="ubuntu_sdkman_${arch}"
local imageid=$(docker images -q $imagename 2> /dev/null)
testcontainers_param="-e TESTCONTAINERS_HOST_OVERRIDE=host.docker.internal"
if [[ -n "$imageid" && -n "$platform" ]]; then
if ! docker image inspect --format "{{.Os}}/{{.Architecture}}" $imageid | grep -i -q -- "${platform#*=}"; then
imageid=""
fi
fi
if [ ! -f $HOME/.bashrc_docker_${arch} ]; then
echo "# bashrc for ptbx_docker_run for ${arch}" >> $HOME/.bashrc_docker_${arch}
fi
if [[ -z "$imageid" ]]; then
docker build $platform --tag $imagename - <<EOT
FROM ubuntu:latest
ARG DEBIAN_FRONTEND=noninteractive
RUN <<'EOS' /bin/bash
set -eux
set -o pipefail
apt-get update
apt-get dist-upgrade -y
apt-get install -y curl zip unzip wget ca-certificates git tig locales netcat-openbsd jq docker.io vim procps less netcat-openbsd dnsutils iputils-ping
locale-gen en_US.UTF-8
groupadd -g $GID mygroup || true
useradd -M -d $HOME -u $UID -g $GID -s /bin/bash $USER
adduser $USER root
EOS
EOT
docker run $platform -e HOME=$HOME -e SDKMAN_DIR=$HOME/.sdkman_docker_${arch} -e GRADLE_USER_HOME=$HOME/.gradle_docker $host_net_param -it --rm -v $HOME:$HOME -u "$UID:${GID:-"$(id -g)"}" -v /var/run/docker.sock:/var/run/docker.sock -v $HOME/.bashrc_docker_${arch}:$HOME/.bashrc -w $PWD $imagename bash -c 'curl -s "https://get.sdkman.io" | bash; source $SDKMAN_DIR/bin/sdkman-init.sh; echo "sdkman_auto_answer=true" >> $SDKMAN_DIR/etc/config; sdk install java 17.0.13-amzn; sdk install maven; sdk install gradle'
fi
docker run $platform --env-file=<(printenv |egrep -v 'SDKMAN|HOME|MANPATH|INFOPATH|PATH') -e HOME=$HOME -e TERM=xterm -e SDKMAN_DIR=$HOME/.sdkman_docker_${arch} -e GRADLE_USER_HOME=$HOME/.gradle_docker -e DOCKER_HOST=unix:///var/run/docker.sock $testcontainers_param --privileged --security-opt seccomp=unconfined --cap-add SYS_ADMIN --cpus=$cpus --memory=$memory $host_net_param -it --rm -u "$UID:${GID:-"$(id -g)"}" --group-add 0 -v $HOME:$HOME -v /var/run/docker.sock:/var/run/docker.sock -v $HOME/.bashrc_docker_${arch}:$HOME/.bashrc -w $PWD $imagename "$@"
else
echo "Unsupported OS: $OSTYPE"
return 1
fi
)
}
function ptbx_docker_run_arm64() {
ptbx_docker_run --platform=linux/arm64 "$@"
}
function ptbx_docker_run_amd64() {
ptbx_docker_run --platform=linux/amd64 "$@"
}
# runs a command with sdkman initialized in the docker container
function ptbx_docker_run_with_sdkman {
local docker_args=()
while [ true ]; do
case "$1" in
--cpus=*)
docker_args+=("$1")
shift
;;
--memory=*)
docker_args+=("$1")
shift
;;
--platform=*)
docker_args+=("$1")
shift
;;
*)
break
;;
esac
done
ptbx_docker_run "${docker_args[@]}" bash -i -c 'source $SDKMAN_DIR/bin/sdkman-init.sh; "$@"' bash "$@"
}
# runs tests with docker to limit cpu & memory, in a loop until it fails
# it is assumed that sdkman is used for JDK management. the default JDK version will be used within docker.
# example: ptbx_until_test_fails_in_docker -Pcore-modules,-main -pl pulsar-broker -Dtest=TopicReaderTest
function ptbx_until_test_fails_in_docker() {
(
local cpus=2
local memory=6g
while [ true ]; do
if [[ "$1" =~ --cpus=.* ]]; then
cpus="${1#*=}"
shift
elif [[ "$1" =~ --memory=.* ]]; then
memory="${1#*=}"
shift
else
break
fi
done
ptbx_docker_run --cpus=$cpus --memory=$memory \
bash -c "source \$HOME/.sdkman/bin/sdkman-init.sh
$(ptbx_until_test_fails_script)" bash "$@"
)
}
function ptbx_until_test_fails_in_docker_with_logs() {
(
ptbx_until_test_fails_in_docker "$@" |& ptbx_tee_log
)
}
function ptbx_until_test_fails() {
(
bash -c "$(ptbx_until_test_fails_script)" bash "$@"
)
}
function ptbx_until_test_fails_with_logs() {
(
ptbx_until_test_fails "$@" |& ptbx_tee_log
)
}
function ptbx_until_test_fails_script() {
cat <<'EOF'
counter=1
while mvn -DredirectTestOutputToFile=false -DtestRetryCount=0 test "$@"; do
echo "----------- LOOP $counter ---------------"
((counter++))
done
echo "Exited after loop #$counter"
EOF
}
function ptbx_run_test() {
(
mvn -DredirectTestOutputToFile=false -DtestRetryCount=0 test "$@"
)
}
function ptbx_run_test_in_docker() {
(
local cpus=2
local memory=6g
local platform=""
while [ true ]; do
if [[ "$1" =~ --cpus=.* ]]; then
cpus="${1#*=}"
shift
elif [[ "$1" =~ --memory=.* ]]; then
memory="${1#*=}"
shift
elif [[ "$1" =~ --platform=.* ]]; then
platform="$1"
shift
else
break
fi
done
ptbx_docker_run --cpus=$cpus --memory=$memory $platform \
bash -c 'source $HOME/.sdkman/bin/sdkman-init.sh; mvn -DredirectTestOutputToFile=false -DtestRetryCount=0 test "$@"' bash "$@"
)
}
function ptbx_run_changed_tests() {
(
local run_all=0
if [[ "$1" == "--all" ]]; then
run_all=1
shift
fi
local compare_to_branch="${1:-"origin/$(ptbx_detect_default_branch)"}"
ptbx_cd_git_root
local root_dir=$(pwd)
local last_module=""
local -a test_classes=()
while read -r file; do
local module=$(echo "$file" | sed 's#/src/.*##g')
if [[ "$module" != "$last_module" && "$last_module" != "" && -n ${test_classes[*]} ]]; then
cd "$root_dir/$last_module"
test_classes=($(printf "%s\n" "${test_classes[@]}" | sort -u))
printf "Running tests in %s for classes:\n" "$last_module"
printf "\t%s\n" "${test_classes[@]}"
ptbx_run_test -Dsurefire.failIfNoSpecifiedTests=false -Dtest="$(IFS=, ; echo "${test_classes[*]}")" || { echo "Failed to run tests in $last_module"; return 1; }
test_classes=()
fi
if [[ "$file" =~ src/test/java/.*Test\.java$ ]]; then
local test_class=$(echo "$file" | sed 's#.*src/test/java/##;s#\.java$##;s#/#.#g')
test_classes+=("$test_class")
elif [[ $run_all == 1 && "$file" =~ src/main/java/.*\.java$ ]]; then
local test_class="$(echo "$file" | sed 's#.*src/main/java/##;s#\.java$##;s#/#.#g')Test"
test_classes+=("$test_class")
fi
last_module="$module"
done < <(git diff --name-only "${compare_to_branch}")
# if test_classes isn't empty
if [[ "$last_module" != "" && -n ${test_classes[*]} ]]; then
cd "$root_dir/$last_module"
test_classes=($(printf "%s\n" "${test_classes[@]}" | sort -u))
printf "Running tests in %s for classes:\n" "$last_module"
printf "\t%s\n" "${test_classes[@]}"
ptbx_run_test -Dsurefire.failIfNoSpecifiedTests=false -Dtest="$(IFS=, ; echo "${test_classes[*]}")" || { echo "Failed to run tests in $last_module"; return 1; }
fi
)
}
function ptbx_build_changed_modules() {
(
local compare_to_branch="${1:-"origin/$(ptbx_detect_default_branch)"}"
ptbx_cd_git_root
changed_modules=$(git diff --name-only "${compare_to_branch}" | grep '/src/' | sed 's#/src/.*##g' | sort -u | tr '\n' ',' | sed 's/,$/\n/')
if [[ -n "$changed_modules" ]]; then
set -x
mvn -pl "$changed_modules" install -DskipTests -Dspotbugs.skip=true
else
echo "No changed modules."
fi
)
}
# prints a date & time up to second resolution
function ptbx_datetime() {
date +%Y-%m-%d-%H%M%S
}
# changes the working directory to the Pulsar source code directory set by PULSAR_DEV_DIR
function ptbx_cd_pulsar_dir {
if [ -n "$PULSAR_DEV_DIR" ]; then
cd "$PULSAR_DEV_DIR"
else
ptbx_cd_git_root
fi
}
function ptbx_cd_git_root {
local gitdir=$(git rev-parse --show-toplevel)
[ ! -d "$gitdir" ] && echo "Not a git directory" && return 1
cd "$gitdir"
}
# creates a local git working directory that can git pull from the actual working directory
# this is useful for running tests in the background
function ptbx_local_clone_create() {
(
set -e
echo "setup local clone"
GITDIR=$(git rev-parse --show-toplevel)
[ ! -d "$GITDIR" ] && echo "Not a git directory" && exit 1
CURRENTBRANCH=$(git rev-parse --abbrev-ref --symbolic-full-name HEAD)
REPONAME=$(basename $GITDIR)
parentdir=$(dirname $GITDIR)
CLONEDIR="$parentdir/$REPONAME.testclone"
[ -d "$CLONEDIR" ] && echo "Clone already exists" && exit 1
git worktree add --detach $CLONEDIR $CURRENTBRANCH
cd "$CLONEDIR"
echo "Clone created in $(pwd)"
)
}
# changes to the "testclone" directory
function ptbx_local_clone_cd() {
ptbx_cd_pulsar_dir
CURRENTBRANCH=$(git rev-parse --abbrev-ref --symbolic-full-name HEAD)
REPONAME=$(basename $PWD)
parentdir=$(dirname $PWD)
CLONEDIR="$parentdir/$REPONAME.testclone"
[ ! -d "$CLONEDIR" ] && ptbx_local_clone_create
cd $CLONEDIR
}
# pushes all changes to repository named "forked"
# useful when calling the github fork of a repository "forked"
function ptbx_gitpush_to_forked() {
ptbx_gitpush_to_remote forked
}
function ptbx_gitpush_to_remote() {
(
remote="${1?remote name required}"
CURRENTBRANCH=$(git rev-parse --abbrev-ref --symbolic-full-name HEAD)
if [ -n "$CURRENTBRANCH" ]; then
git push -f "$remote" "$CURRENTBRANCH:$CURRENTBRANCH"
fi
)
}
# pushes changes to a PR branch to the forked repository
# use this when you have a PR branch checked out with
# command "gh pr checkout <pr-number>" and want to push changes to it
function ptbx_gitpush_to_pr_branch() {
(
PR_NUMBER="$1"
if [ -z "$PR_NUMBER" ]; then
echo "Pass PR number as argument"
return 1
fi
SLUG=$(ptbx_gh_slug origin)
FORK_REPO=$(curl -s https://api.github.com/repos/$SLUG/pulls/$PR_NUMBER | jq -r '.head.repo.html_url')
FORK_BRANCH=$(curl -s https://api.github.com/repos/$SLUG/pulls/$PR_NUMBER | jq -r '.head.ref')
if [ -z "$FORK_REPO" ]; then
echo "Cannot find forked repo for PR $PR_NUMBER"
return 1
fi
git push "$FORK_REPO" "HEAD:$FORK_BRANCH"
)
}
# synchronizes the forked/master remote branch with origin/master
function ptbx_git_sync_forked_master_with_upstream() {
(
git fetch origin
local default_branch=$(ptbx_detect_default_branch)
git update-ref refs/heads/${default_branch} origin/${default_branch}
git push -f forked ${default_branch}
)
}
function _ptbx_git_sync_branches() {
(
local remote=$1
shift
local -a branches=("${@}")
git fetch $remote
for branch in "${branches[@]}"; do
local current_branch="$(git branch --show-current)"
if [[ $branch == $current_branch ]]; then
git checkout --detach HEAD
fi
git update-ref refs/heads/$branch $remote/$branch
if [[ $branch == $current_branch ]]; then
git checkout $branch
fi
done
)
}
function ptbx_git_sync_pulsar_maintenance_branches_with_upstream() {
(
cd ~/workspace-pulsar/pulsar
_ptbx_git_sync_branches origin master branch-2.7 branch-2.8 branch-2.9 branch-2.10 branch-2.11 branch-3.0
cd ~/workspace-pulsar/pulsar.datastax
_ptbx_git_sync_branches datastax 2.7.2_ds 2.8.0_ds 2.8.3_ds 2.10_ds
)
}
# generates ssh config file for connecting to running vms managed by https://multipass.run/
# this is useful for using rsync to copy files to/from multipass vm
# prerequisite: copy the multipass ssh key:
# sudo cp /var/snap/multipass/common/data/multipassd/ssh-keys/id_rsa ~/.ssh/multipass_id_rsa
# sudo chown $USER:$GROUP ~/.ssh/multipass_id_rsa
# ssh-keygen -y -f ~/.ssh/multipass_id_rsa > ~/.ssh/multipass_id_rsa.pub
function ptbx_multipass_update_sshconfig() {
(
echo 'Host *.multipass
User ubuntu
IdentityFile ~/.ssh/multipass_id_rsa
IdentitiesOnly yes
UserKnownHostsFile /dev/null
StrictHostKeyChecking no
PasswordAuthentication no
'
IFS='
'
for vm in $(multipass ls --format csv | grep Running); do
echo "Host $(echo $vm | awk -F , '{ print $1 }').multipass
Hostname $(echo $vm | awk -F , '{ print $3 }')
"
done
) >~/.ssh/multipass_ssh_config
echo 'Updated ~/.ssh/sshconfig_multipass. use "Include ~/.ssh/multipass_ssh_config" to include it in ~/.ssh/config'
}
# creates a multipass vm and installs docker in it
function ptbx_multipass_create_vm_with_docker() {
local vmname="$1"
[ -n "$vmname" ] || {
echo "Pass VM name as argument"
return 1
}
(
multipass launch -d 20G -n $vmname
echo 'export DEBIAN_FRONTEND=noninteractive
sudo apt-get update
sudo apt-get -y install docker.io
sudo adduser ubuntu docker
' | multipass shell $vmname
)
}
function ptbx_multipass_copy_ssh_key() {
(
sudo cp /var/snap/multipass/common/data/multipassd/ssh-keys/id_rsa ~/.ssh/multipass_id_rsa
sudo chown $USER:$GROUP ~/.ssh/multipass_id_rsa
chmod 0600 ~/.ssh/multipass_id_rsa
ssh-keygen -y -f ~/.ssh/multipass_id_rsa >~/.ssh/multipass_id_rsa.pub
)
}
# workaround for https://github.com/canonical/multipass/issues/1866
function ptbx_multipass_fix_network() {
(
for table in filter nat mangle; do
sudo iptables-legacy -t $table -S | grep Multipass | xargs -L1 sudo iptables-nft -t $table
done
)
}
function ptbx_multipass_delete() {
(
local name="$1"
multipass stop "$name" && multipass delete "$name" && multipass purge
)
}
# changing cpus or memory for multipass vm
# https://github.com/canonical/multipass/issues/1158#issuecomment-577315005
function ptbx_multipass_edit_config() {
sudo bash -c 'systemctl stop snap.multipass.multipassd.service;vi /var/snap/multipass/common/data/multipassd/multipassd-vm-instances.json;systemctl start snap.multipass.multipassd.service'
}
# uploads a maven build log file as a gist, converting to plain text (remove ansi code)
function ptbx_upload_log_to_gist() {
(
local filename="$1"
shift
if [ ! -f "$filename" ]; then
echo "File '${filename}' doesn't exist."
echo "usage: ptbx_upload_log_to_gist [filename] -d [description]"
exit 1
fi
cat "$filename" | ansi2txt >"${filename}.txt"
gh gist create "${filename}.txt" "$@"
)
}
function ptbx_project_version() {
if command -v xmlstarlet &>/dev/null; then
# fast way to extract project version
xmlstarlet sel -t -m _:project -v _:version -n pom.xml
else
# prints out the project version and nothing else
# https://maven.apache.org/plugins/maven-help-plugin/evaluate-mojo.html#forceStdout
mvn initialize help:evaluate -Dexpression=project.version -pl . -q -DforceStdout | sed 's/\[INFO\] \[stdout\] //' | grep -F -v '[WARN]' | tail -1
fi
}
function ptbx_build_docker_pulsar_all_image() {
(
docker pull ubuntu:20.04
ptbx_clean_cppbuild
command mvn clean install -Dspotbugs.skip=true -DskipTests
command mvn -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests
command mvn -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests
)
}
function ptbx_build_test_latest_version_image() {
(
ptbx_build_docker_pulsar_all_image
command mvn -B -f tests/docker-images/pom.xml install -am -Pdocker -Dspotbugs.skip=true -DskipTests
)
}
function ptbx_build_pulsar_all_and_push_to_microk8s() {
(
ptbx_build_and_push_pulsar_images localhost:32000/apachepulsar
)
}
function ptbx_build_and_push_pulsar_images() {
(
ptbx_build_docker_pulsar_all_image || return 1
ptbx_push_pulsar_images "$@"
)
}
function ptbx_push_pulsar_images() {
(
docker_repo_prefix=${1:-"$PTBX_DEFAULT_DOCKER_REPO_PREFIX"}
docker_tag="$2"
docker_repo_image_prefix=${3:-"$PTBX_DEFAULT_DOCKER_IMAGE_PREFIX"}
if [[ -z "$docker_tag" ]]; then
gitrev=$(git rev-parse HEAD | colrm 10)
project_version=$(ptbx_project_version)
docker_tag="${project_version}-$gitrev"
fi
set -xe
docker tag apachepulsar/pulsar-all:latest ${docker_repo_prefix}/${docker_repo_image_prefix}-all:${docker_tag}
docker tag apachepulsar/pulsar:latest ${docker_repo_prefix}/${docker_repo_image_prefix}:${docker_tag}
docker push ${docker_repo_prefix}/${docker_repo_image_prefix}-all:${docker_tag}
docker push ${docker_repo_prefix}/${docker_repo_image_prefix}:${docker_tag}
)
}
function ptbx_push_pulsar_all_with_openid_connect_plugin() {
(
docker_repo_prefix=${1:-"$PTBX_DEFAULT_DOCKER_REPO_PREFIX"}
docker_tag="$2"
docker_repo_image_prefix=${3:-"$PTBX_DEFAULT_DOCKER_IMAGE_PREFIX"}
TEMP_DIR="$(mktemp -d)"
mkdir "$TEMP_DIR/extra-jars"
cd "$TEMP_DIR/extra-jars"
curl -L -O https://github.com/datastax/pulsar-openid-connect-plugin/releases/download/1.0.0-beta/pulsar-openid-connect-plugin-1.0.0-beta.jar
cd "$TEMP_DIR"
cat >Dockerfile <<EOF
FROM apachepulsar/pulsar-all:latest
# COPY extra jars
COPY --chown=pulsar:0 extra-jars/*.jar /pulsar/lib/
EOF
docker build -t ${docker_repo_prefix}/${docker_repo_image}:${docker_tag} .
docker push ${docker_repo_prefix}/${docker_repo_image}:${docker_tag}
)
}
function ptbx_build_java_test_image() {
(
docker pull ubuntu:22.04
ptbx_cd_git_root
./build/build_java_test_image.sh -Dcheckstyle.skip=true || return 1
)
}
function ptbx_build_and_push_java_test_image_to_microk8s() {
(
ptbx_build_and_push_java_test_image localhost:32000/apachepulsar
)
}
function ptbx_build_and_push_java_test_image() {
(
ptbx_build_java_test_image
docker_repo_prefix=${1:-"$PTBX_DEFAULT_DOCKER_REPO_PREFIX"}
gitrev=$(git rev-parse HEAD | colrm 10)
project_version=$(ptbx_project_version)
docker_tag="${project_version}-$gitrev"
set -xe
docker tag apachepulsar/java-test-image:latest ${docker_repo_prefix}/${PTBX_DEFAULT_JAVA_TEST_IMAGE_NAME}:${docker_tag}
docker push ${docker_repo_prefix}/${PTBX_DEFAULT_JAVA_TEST_IMAGE_NAME}:${docker_tag}
)
}
function ptbx_forked_repo() {
ptbx_gh_slug forked
}
function ptbx_gh_slug() {
local repo="$(git remote get-url "$1")"
repo="${repo##*github.com/}"
repo="${repo%.*}"
echo "$repo"
}
function ptbx_github_open_pr_to_own_fork() {
local default_branch=$(ptbx_detect_default_branch)
gh pr create "--repo=$(ptbx_forked_repo)" --base "${default_branch}" --head "$(git branch --show-current)" -f "$@"
}
function ptbx_detect_default_branch() {
if [[ "$(git branch --list -r origin/main | wc -l)" == "1" ]]; then
echo main
else
echo master
fi
}
function ptbx_github_open_pr() {
local github_user="$(ptbx_forked_repo)"
github_user="${github_user%/*}"
local default_branch=$(ptbx_detect_default_branch)
gh pr create "--repo=$(ptbx_gh_slug origin)" --base "${default_branch}" --head "$github_user:$(git branch --show-current)" -w
}
function ptbx_github_test_pr_in_own_fork() {
local github_user="$(ptbx_forked_repo)"
github_user="${github_user%/*}"
local pr_json=$(curl -s "https://api.github.com/repos/$(ptbx_gh_slug origin)/pulls?head=${github_user}:$(git branch --show-current)" |jq '.[0]')
if printf "%s" "${pr_json}" | jq --arg github_user "${github_user}" -e 'select(.user.login == $github_user)' &> /dev/null; then
local fork_pr_title=$(printf "%s" "${pr_json}" | jq -r '"[run-tests] " + .title')
local pr_url=$(printf "%s" "${pr_json}" | jq -r '.html_url')
local fork_pr_body="This PR is for running tests for upstream PR ${pr_url}."
ptbx_git_sync_forked_master_with_upstream
ptbx_github_open_pr_to_own_fork -b "${fork_pr_body}" -t "${fork_pr_title}"
local fork_pr_json=$(curl -s "https://api.github.com/repos/$(ptbx_forked_repo)/pulls?head=$(git branch --show-current)" |jq '.[0]')
local fork_pr_url=$(printf "%s" "${fork_pr_json}" | jq -r '.html_url')
if [ -n "${fork_pr_url}" ]; then
local pr_body_updated=$(printf "%s" "${pr_json}" | jq --arg fork_pr_url "${fork_pr_url}" -r '.body | sub("(?s)<!-- ENTER URL HERE.*?-->";$fork_pr_url)')
if [ -n "${pr_body_updated}" ]; then
gh pr edit "${pr_url}" --body "${pr_body_updated}"
fi
fi
else
echo "Cannot find PR for current branch."
fi
}
function ptbx_reset_iptables() {
(
sudo su <<'EOF'
for iptables_bin in iptables iptables-legacy; do
$iptables_bin -F
$iptables_bin -X
$iptables_bin -t nat -F
$iptables_bin -t nat -X
$iptables_bin -t mangle -F
$iptables_bin -t mangle -X
$iptables_bin -P INPUT ACCEPT
$iptables_bin -P FORWARD ACCEPT
$iptables_bin -P OUTPUT ACCEPT
done
EOF
)
}
# shows the tracking branch name, usually origin/master
function ptbx_git_upstream_branch() {
local default_branch=$(ptbx_detect_default_branch)
git rev-parse --abbrev-ref "${default_branch}@{upstream}"
}
# soft resets all changes to the upstream, useful for re-committing changes in a branch
function ptbx_reset_to_merge_base_in_upstream() {
git reset "$(git merge-base $(ptbx_git_upstream_branch) HEAD)" "$@"
}
# helper for re-committing changes in a branch
# this is an alternative way for squashing commits
# prerequisite is to first reset all commits in the branch with ptbx_reset_to_merge_base
function ptbx_commit_based_on_reference() {
local sha=$1
# add same files
git log --format="" -n 1 --stat --name-only $sha | xargs git add -f
# copy commit message
git commit --no-edit --reuse-message=$sha "$@"
}
function _ptbx_extract_threaddumps() {
ansi2txt | colrm 1 29 | csplit - -f threadump$(date -I)_ -b %02d.txt --suppress-matched -z '/----------------------- pid/' '{*}'
}
function ptbx_extract_threaddumps() {
local FILE="${1:-"$(find -name "*_print\ JVM\ thread\ dumps\ when\ cancelled.txt" -print -quit)"}"
cat "$FILE" | _ptbx_extract_threaddumps
}
function ptbx_extract_threaddumps_from_zip() {
local ZIPFILE=$1
unzip -p $ZIPFILE "*print JVM thread dumps*" | _ptbx_extract_threaddumps
}
function ptbx_extract_threaddumps_from_file() {
local prefix="threaddump${RANDOM}_$(date -I)_"
cat "$1" | awk '{sub(/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+Z /, "")} /^Full thread dump OpenJDK/{p=1; dump_count++} /^----------------------- pid/{if(p){print "DUMPSEPARATOR";p=0;if(dump_count>1)next}} /^[[:space:]]*class space/{print;if(dump_count>1)exit} p' | csplit - -f $prefix -b %02d.txt -z --suppress-matched '/DUMPSEPARATOR/' '{*}'
for file in ${prefix}*; do
if ! grep -q "Full thread dump OpenJDK" $file; then
rm $file
fi
done
}
function ptbx_find_created_threads_in_test_logs() {
grep -shEr "created [0-9]+ new threads" * | awk -F "Summary: " '{ print $2 }' | awk '{ print $(NF-2), $0}' |sort -rn | cut -f2- -d' '
}
function ptbx_search_jars() {
(
if [ $ZSH_VERSION ]; then
setopt sh_word_split
fi
IFS=$'\n'
JARFILES=$(find -name "*.jar")
for i in $JARFILES; do
RESULTS=$(unzip -Z -1 -C "$i" "$1" 2>/dev/null)
if [ -n "$RESULTS" ]; then
echo Results in $i
echo "${RESULTS}"
fi
done
)
}
function ptbx_cherrypick_branch() {
(
set -xe
branch="$1"
target_branch="$2"
source_trunk="${3:-master}"
temp_branch="tempbranch$$"
git branch "$temp_branch" "$branch"
git rebase --onto "${target_branch}" "$(git merge-base "${source_trunk}" "${branch}")" "${temp_branch}"
git checkout "${target_branch}"
git merge --squash "$temp_branch"
git branch -D "$temp_branch"
)
}
function ptbx_mvn_publish_to_apache_repository() {
(
[ -f ~/.m2/apache-settings.xml ] || curl -o ~/.m2/apache-settings.xml https://raw.githubusercontent.com/apache/pulsar/master/src/settings.xml
export APACHE_USER="${APACHE_USER:-$USER}"
stty -echo
printf "Password: "
read APACHE_PASSWORD
stty echo
printf "\n"
export APACHE_PASSWORD
export GPG_TTY=$(tty)
mvn deploy -DskipTests --settings ~/.m2/apache-settings.xml
)
}
function ptbx_copy_docker_image_to_microk8s() {
(
source_image=$1
target_image=localhost:32000/$1
docker tag $source_image $target_image
docker push $target_image
CTR="ctr -a /var/snap/microk8s/common/run/containerd.sock -n k8s.io"
sudo bash -c "$CTR images pull --plain-http $target_image && $CTR images tag $target_image $source_image docker.io/$source_image"
)
}
function ptbx_kubectl_check_auth() {
(
for resource in "$@"; do
for verb in create get list watch update patch delete deletecollection; do
echo "$resource $verb $(kubectl auth can-i $verb $resource)"
done
done
)
}
function ptbx_list_images_in_ds_pulsar_values() {
(
values_file="$HOME/workspace-datastax/datastax-pulsar-helm-chart/helm-chart-sources/pulsar/values.yaml"
yq e '.image | .[] |= ([.repository, .tag] | join(":")) | to_entries | .[] | .value' "$values_file" | sort | uniq
)
}
function ptbx_crc_ssh() {
ssh -i "$HOME/.crc/machines/crc/id_ecdsa" core@192.168.130.11 "$@"
}
function ptbx_copy_docker_image_to_crc() {
(
source_image="$1"
target_image="default-route-openshift-image-registry.apps-crc.testing/$(oc project -q)/${source_image#*/}"
docker tag "$source_image" "$target_image"
docker login -u kubeadmin -p "$(oc whoami -t)" default-route-openshift-image-registry.apps-crc.testing
docker push "$target_image"
ptbx_crc_ssh sudo podman login -u kubeadmin -p "$(oc whoami -t)" default-route-openshift-image-registry.apps-crc.testing --tls-verify=false
ptbx_crc_ssh sudo podman image pull --tls-verify=false "$target_image"
ptbx_crc_ssh sudo podman image tag "$target_image" "$source_image"
ptbx_crc_ssh sudo podman image tag "$target_image" "docker.io/$source_image"
)
}
function ptbx_copy_ds_helm_chart_images_to_crc() {
(
for image in $(ptbx_list_images_in_ds_pulsar_values); do
ptbx_copy_docker_image_to_crc "$image"
done
)
}
function _ptbx_upload_encrypted() {
local file_name="$1"
local recipient="$2"
gpg -k "$recipient" &> /dev/null || gpg --recv-key "$recipient" &> /dev/null || { echo "Searching for key for $recipient"; gpg --search-keys "$recipient"; }
local transfer_url=$(gpg --encrypt --recipient "$recipient" --trust-model always \
|curl --progress-bar --upload-file "-" "https://transfer.sh/${file_name}.gpg" \
|tee /dev/null)
echo ""
echo "command for receiving: curl $transfer_url | gpg --decrypt > ${file_name}"
}
function ptbx_transfer(){
if [ "$1" == "--desc" ]; then
echo "Transfers files with gpg encryption over transfer.sh"
return 0
fi
if [ $# -lt 2 ]; then
echo "No arguments specified.\nUsage:\n ptbx_transfer <file|directory> recipient\n ... | ptbx_transfer <file_name> recipient">&2
return 1
fi
if tty -s; then
local file="$1"
local recipient="$2"
local file_name=$(basename "$file")
if [ ! -e "$file" ]; then
echo "$file: No such file or directory">&2
return 1
fi
if [ -d "$file" ]; then
file_name="${file_name}.tar.gz"
tar zcf - "$file" | _ptbx_upload_encrypted $file_name $recipient
else
cat "$file" | _ptbx_upload_encrypted $file_name $recipient
fi
else
local file_name=$1
local recipient="$2"
_ptbx_upload_encrypted $file_name $recipient
fi
}
function ptbx_transfer_unencrypted() {
if [ $# -eq 0 ];then
echo "No arguments specified.\nUsage:\n ptbx_transfer_unencrypted <file|directory>\n ... | ptbx_transfer_unencrypted <file_name>">&2