-
Notifications
You must be signed in to change notification settings - Fork 0
/
Jenkinsfile-update-gitcache
150 lines (133 loc) · 7.2 KB
/
Jenkinsfile-update-gitcache
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
// A job for regular re-fetching of reference repository cache
// for (shared) Git consumer usage.
// The git reference repository should usefully be accessible
// from the system that does git checkouts; in thightly-knit
// CI farms this may be an NFS share accessible to everyone.
// Even then, it is much more efficient to run git fetch on
// such NFS server "locally" (well, using SSH with trusted
// keys pre-set in jenkins user, or credentials integrated
// below [TODO]), than over a remote filesystem.
// It may help if that worker has proxy settings (for https
// and/or ssh-over-https)
// By default, we tap into the Jenkins controller itself:
def gitcache_owner = "jenkins"
def gitcache_server = "localhost"
def gitcache_dir = "/var/lib/jenkins/gitcache"
def gitcache_lockname = "gitcache:jenkins-controller"
def gitcache_proxyfile = "/etc/proxy.default"
pipeline {
agent { label "master-infra" }
parameters {
choice (
name: 'FETCH_PARALLEL',
choices: ['default-par', 'default-seq', 'default', 'true', 'false'],
description: '''Run many git processes at once - less wallclock time, potentially higher risk of failure due to resource contention, and accessing same lock files (especially for same-named tags processing in different remotes from different processes).\n
The "default-par" mode is to drill into sub-refrepos and parallelize fetches in those, while also processing such subdirectories in a parallel manner.\n
The "default-seq" mode is to drill into sub-refrepos and parallelize fetches in those, while processing such subdirectories sequentially.\n
The "true" parallel mode builds a list and spawns many git clients at once, and may be unsafe (to be proven).\n
The "false" mode builds the list and passes each fetch one by one, slowest but surest.'''
)
booleanParam (
name: 'DEBUG',
defaultValue: false,
description: 'Enable to trace the register-git-cache.sh script run'
)
booleanParam (
name: 'CI_TIME',
defaultValue: false,
description: 'Report more data for time-profiling the script'
)
booleanParam (
name: 'QUIET_SKIP',
defaultValue: true,
description: 'Announce every skipped already-existing repo, etc.?'
)
booleanParam (
name: 'CI_TIME_LOOPS',
defaultValue: false,
description: 'Heavier profiling of intensive loops - more data logged, but more impact from measurement itself.'
)
}
options {
disableConcurrentBuilds()
disableResume()
durabilityHint('PERFORMANCE_OPTIMIZED')
buildDiscarder(logRotator(numToKeepStr: '10'))
// Note: we presume here that the location for refrepos
// is already prepared, including the base dir for the
// fanout of many repos, with (a symlink to) the script
// we use for actual work - register-git-cache.sh from
// https://github.com/jimklimov/git-refrepo-scripts
skipDefaultCheckout true
// https://plugins.jenkins.io/timestamper
timestamps()
// https://github.com/jenkinsci/ansicolor-plugin
ansiColor('xterm')
}
triggers {
cron('H/30 * * * *')
}
stages {
stage("Updater") {
steps {
timeout (time: 9600, unit: 'SECONDS', activity: true) {
lock (resource: gitcache_lockname, quantity: 1) {
sh """
### Avoid erroring out
set +e
set -x
### More generic/failoverish to do it over NFS
cd '${gitcache_dir}' || exit \$? ### Even in SSH below, better note that NFS resource is accessible and server deems itself running
#git fetch --all
#time REFREPODIR_MODE=GIT_SUBMODULES ./register-git-cache.sh fetch -vp
### Faster to do it locally on the storage host than over NFS from a worker
### Note: shell-session logins should include the default proxy settings for HTTP(S) sources
RES=0
DO_FETCH='default'
case "\$FETCH_PARALLEL" in
default)
DO_FETCH='default'
;;
default-par)
DO_FETCH='-p'
;;
default-seq)
DO_FETCH='-s'
;;
true)
DO_FETCH='-vp'
;;
false)
DO_FETCH='-vs'
;;
esac
time ssh "${gitcache_owner}@${gitcache_server}" "if [ -n '${gitcache_proxyfile}' ] && [ -s '${gitcache_proxyfile}' ]; then . '${gitcache_proxyfile}' || true ; fi; cd '${gitcache_dir}' && DEBUG='${params.DEBUG}' QUIET_SKIP='${params.QUIET_SKIP}' CI_TIME='${params.CI_TIME}' DO_FETCH='${params.DO_FETCH}' REFREPODIR_MODE='GIT_SUBMODULES' ./register-git-cache.sh add-recursive all" || RES=\$?
exit \$RES
###### LEGACY SCRIPT:
case "\$FETCH_PARALLEL" in
default)
#ssh -t -t "${gitcache_owner}@${gitcache_server}" "bash --login -c 'cd '${gitcache_dir}' && time ./register-git-cache.sh fetch'"
time ssh "${gitcache_owner}@${gitcache_server}" "if [ -n '${gitcache_proxyfile}' ] && [ -s '${gitcache_proxyfile}' ]; then . '${gitcache_proxyfile}' || true ; fi; cd '${gitcache_dir}' && DEBUG='${params.DEBUG}' QUIET_SKIP='${params.QUIET_SKIP}' CI_TIME='${params.CI_TIME}' REFREPODIR_MODE='GIT_SUBMODULES' ./register-git-cache.sh fetch" || RES=\$?
;;
true)
#ssh -t -t "${gitcache_owner}@${gitcache_server}" "bash --login -c 'cd '${gitcache_dir}' && time ./register-git-cache.sh fetch -vp'"
time ssh "${gitcache_owner}@${gitcache_server}" "if [ -n '${gitcache_proxyfile}' ] && [ -s '${gitcache_proxyfile}' ]; then . '${gitcache_proxyfile}' || true ; fi; cd '${gitcache_dir}' && DEBUG='${params.DEBUG}' QUIET_SKIP='${params.QUIET_SKIP}' CI_TIME='${params.CI_TIME}' REFREPODIR_MODE='GIT_SUBMODULES' ./register-git-cache.sh fetch -vp" || RES=\$?
;;
false)
#ssh -t -t "${gitcache_owner}@${gitcache_server}" "bash --login -c 'cd '${gitcache_dir}' && time ./register-git-cache.sh fetch -vs'"
time ssh "${gitcache_owner}@${gitcache_server}" "if [ -n '${gitcache_proxyfile}' ] && [ -s '${gitcache_proxyfile}' ]; then . '${gitcache_proxyfile}' || true ; fi; cd '${gitcache_dir}' && DEBUG='${params.DEBUG}' QUIET_SKIP='${params.QUIET_SKIP}' CI_TIME='${params.CI_TIME}' REFREPODIR_MODE='GIT_SUBMODULES' ./register-git-cache.sh fetch -vs" || RES=\$?
;;
esac
### By default this does a fetch of all repos to see their branches up to date
### so we pass a DO_FETCH=false to avoid re-fetching earlier known repos that
### we have just updated above
#time ssh "${gitcache_owner}@${gitcache_server}" "if [ -n '${gitcache_proxyfile}' ] && [ -s '${gitcache_proxyfile}' ]; then . '${gitcache_proxyfile}' || true ; fi; cd '${gitcache_dir}' && DO_FETCH=false DEBUG='${params.DEBUG}' REFREPODIR_MODE=GIT_SUBMODULES ./register-git-cache.sh add-recursive `SKIP_LOCK=true REFREPODIR_MODE='GIT_SUBMODULES' ./register-git-cache.sh ls | awk '{print \$2}' | tr '\n' ' '`" || RES=\$?
time ssh "${gitcache_owner}@${gitcache_server}" "if [ -n '${gitcache_proxyfile}' ] && [ -s '${gitcache_proxyfile}' ]; then . '${gitcache_proxyfile}' || true ; fi; cd '${gitcache_dir}' && DO_FETCH=false DEBUG='${params.DEBUG}' QUIET_SKIP='${params.QUIET_SKIP}' CI_TIME='${params.CI_TIME}' REFREPODIR_MODE='GIT_SUBMODULES' ./register-git-cache.sh add-recursive all" || RES=\$?
exit \$RES
"""
} // lock
} // timeout
} // steps
} // stage Updater
} // stages
}