forked from crc-org/snc
-
Notifications
You must be signed in to change notification settings - Fork 0
/
snc-library.sh
executable file
·298 lines (253 loc) · 13.4 KB
/
snc-library.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
#!/bin/bash
set -exuo pipefail
function preflight_failure() {
local msg=$1
echo "$msg"
if [ -z "${SNC_NON_FATAL_PREFLIGHT_CHECKS-}" ]; then
exit 1
fi
}
function check_oc_version() {
local current_oc_version=
if [ -f "$OC" ]; then
current_oc_version=$(${OC} version --client -o json |jq -r .releaseClientVersion)
fi
[ ${current_oc_version} = ${OPENSHIFT_RELEASE_VERSION} ]
}
function download_oc() {
local current_oc_version=
if [ -f "$OC" ]; then
current_oc_version=$(${OC} version --client -o json |jq -r .releaseClientVersion)
if [ ${current_oc_version} = ${OPENSHIFT_RELEASE_VERSION} ]; then
echo "No need to download oc, local oc is already version ${OPENSHIFT_RELEASE_VERSION}"
return
fi
fi
mkdir -p openshift-clients/linux
curl -L "${MIRROR}/${OPENSHIFT_RELEASE_VERSION}/openshift-client-linux-${OPENSHIFT_RELEASE_VERSION}.tar.gz" | tar -zx -C openshift-clients/linux oc
if [ "${SNC_GENERATE_MACOS_BUNDLE}" != "0" ]; then
mkdir -p openshift-clients/mac
curl -L "${MIRROR}/${OPENSHIFT_RELEASE_VERSION}/openshift-client-mac-${OPENSHIFT_RELEASE_VERSION}.tar.gz" | tar -zx -C openshift-clients/mac oc
fi
if [ "${SNC_GENERATE_WINDOWS_BUNDLE}" != "0" ]; then
mkdir -p openshift-clients/windows
curl -L "${MIRROR}/${OPENSHIFT_RELEASE_VERSION}/openshift-client-windows-${OPENSHIFT_RELEASE_VERSION}.zip" > openshift-clients/windows/oc.zip
${UNZIP} -o -d openshift-clients/windows/ openshift-clients/windows/oc.zip
fi
}
function run_preflight_checks() {
local bundle_type=$1
if [ -z "${OPENSHIFT_PULL_SECRET_PATH-}" ]; then
echo "OpenShift pull secret file path must be specified through the OPENSHIFT_PULL_SECRET_PATH environment variable"
exit 1
elif [ ! -f ${OPENSHIFT_PULL_SECRET_PATH} ]; then
echo "Provided OPENSHIFT_PULL_SECRET_PATH (${OPENSHIFT_PULL_SECRET_PATH}) does not exists"
exit 1
fi
echo "Checking libvirt and DNS configuration"
LIBVIRT_URI=qemu:///system
if [ ${bundle_type} == "snc" ] || [ ${bundle_type} == "okd" ]; then
LIBVIRT_URI=qemu+tcp://localhost/system
fi
# check if we can connect to ${LIBVIRT_URI}
if ! sudo virsh -c ${LIBVIRT_URI} uri >/dev/null; then
preflight_failure "libvirtd is not listening for ${LIBVIRT_URI}, see https://github.com/openshift/installer/tree/master/docs/dev/libvirt#configure-libvirt-to-accept-tcp-connections"
fi
if ! sudo virsh -c ${LIBVIRT_URI} net-info default &> /dev/null; then
echo "Installing libvirt default network configuration"
sudo dnf install -y libvirt-daemon-config-network || exit 1
fi
echo "default libvirt network is available"
#Check if default libvirt network is Active
if [[ $(sudo virsh -c ${LIBVIRT_URI} net-info default | awk '{print $2}' | sed '3q;d') == "no" ]]; then
echo "Default network is not active, starting it"
sudo virsh -c ${LIBVIRT_URI} net-start default || exit 1
fi
#Just warn if architecture is not supported
case $ARCH in
x86_64|ppc64le|s390x|aarch64)
echo "The host arch is ${ARCH}.";;
*)
echo "The host arch is ${ARCH}. This is not supported by SNC!";;
esac
# check for availability of a hypervisor using kvm
if ! sudo virsh -c ${LIBVIRT_URI} capabilities | ${XMLLINT} --xpath "/capabilities/guest/arch[@name='${ARCH}']/domain[@type='kvm']" - &>/dev/null; then
preflight_failure "Your ${ARCH} platform does not provide a hardware-accelerated hypervisor, it's strongly recommended to enable it before running SNC. Check virt-host-validate for more detailed diagnostics"
return
fi
if [ ${bundle_type} == "snc" ] || [ ${bundle_type} == "okd" ]; then
# check that api.${SNC_PRODUCT_NAME}.${BASE_DOMAIN} either can't be resolved, or resolves to 192.168.126.1[01]
local ping_status
ping_status="$(ping -c1 api.${SNC_PRODUCT_NAME}.${BASE_DOMAIN} | head -1 || true >/dev/null)"
if echo ${ping_status} | grep "PING api.${SNC_PRODUCT_NAME}.${BASE_DOMAIN} (" && ! echo ${ping_status} | grep "192.168.126.1[01])"; then
preflight_failure "DNS setup seems wrong, api.${SNC_PRODUCT_NAME}.${BASE_DOMAIN} resolved to an IP which is neither 192.168.126.10 nor 192.168.126.11, please check your NetworkManager configuration and /etc/hosts content"
return
fi
fi
# check if firewalld is configured to allow traffic from 192.168.126.0/24 to 192.168.122.1
# this check is very basic and expects the configuration to match
# https://github.com/openshift/installer/tree/master/docs/dev/libvirt#firewalld
# Disabled for now as on stock RHEL8 installs, additional permissions are needed for
# firewall-cmd --list-services, so this test fails for unrelated reasons
#
#local zone
#if firewall-cmd -h >/dev/null; then
# # With older libvirt, the 'libvirt' zone will not exist
# if firewall-cmd --get-zones |grep '\<libvirt\>'; then
# zone=libvirt
# else
# zone=dmz
# fi
# if ! firewall-cmd --zone=${zone} --list-services | grep '\<libvirt\>'; then
# preflight_failure "firewalld is available, but it is not configured to allow 'libvirt' traffic in either the 'libvirt' or 'dmz' zone, please check https://github.com/openshift/installer/tree/master/docs/dev/libvirt#firewalld"
# return
# fi
#fi
echo "libvirt and DNS configuration successfully checked"
}
function replace_pull_secret() {
# Hide the output of 'cat $OPENSHIFT_PULL_SECRET_PATH' so that it doesn't
# get leaked in CI logs
set +x
local filename=$1
pull_secret="$(< $OPENSHIFT_PULL_SECRET_PATH)" ${YQ} eval --inplace '.pullSecret = strenv(pull_secret) | .pullSecret style="literal"' $filename
set -x
}
function create_json_description {
local bundle_type=$1
sncGitHash=$(git describe --abbrev=4 HEAD 2>/dev/null || git rev-parse --short=4 HEAD)
echo {} | ${JQ} '.version = "1.4"' \
| ${JQ} ".type = \"${BUNDLE_TYPE}\"" \
| ${JQ} ".arch = \"${yq_ARCH}\"" \
| ${JQ} ".buildInfo.buildTime = \"$(date -u --iso-8601=seconds)\"" \
| ${JQ} ".buildInfo.sncVersion = \"git${sncGitHash}\"" \
| ${JQ} ".clusterInfo.openshiftVersion = \"${OPENSHIFT_RELEASE_VERSION}\"" \
| ${JQ} ".clusterInfo.clusterName = \"${SNC_PRODUCT_NAME}\"" \
| ${JQ} ".clusterInfo.baseDomain = \"${BASE_DOMAIN}\"" \
| ${JQ} ".clusterInfo.appsDomain = \"apps-${SNC_PRODUCT_NAME}.${BASE_DOMAIN}\"" >${INSTALL_DIR}/crc-bundle-info.json
if [ ${bundle_type} == "snc" ] || [ ${bundle_type} == "okd" ]; then
openshiftInstallerVersion=$(${OPENSHIFT_INSTALL} version)
tmp=$(mktemp)
cat ${INSTALL_DIR}/crc-bundle-info.json | ${JQ} ".buildInfo.openshiftInstallerVersion = \"${openshiftInstallerVersion}\"" \
> ${tmp} && mv ${tmp} ${INSTALL_DIR}/crc-bundle-info.json
fi
}
function create_pvs() {
local bundle_type=$1
# Create hostpath-provisioner namespace
retry ${OC} apply -f kubevirt-hostpath-provisioner-csi/namespace.yaml
# Add external provisioner RBACs
retry ${OC} apply -f kubevirt-hostpath-provisioner-csi/external-provisioner-rbac.yaml -n hostpath-provisioner
# Create CSIDriver/kubevirt.io.hostpath-provisioner resource
retry ${OC} apply -f kubevirt-hostpath-provisioner-csi/csi-driver-hostpath-provisioner.yaml -n hostpath-provisioner
# Apply SCC allowin hostpath-provisioner containers to run as root and access host network
retry ${OC} apply -f kubevirt-hostpath-provisioner-csi/kubevirt-hostpath-security-constraints-csi.yaml
# Deploy csi driver components
if [[ ${bundle_type} == "snc" ]]; then
# in case of OCP we want the images to come from registry.redhat.io
# this is done using the kustomize.yaml file
retry ${OC} apply -k kubevirt-hostpath-provisioner-csi/csi-driver -n hostpath-provisioner
else
retry ${OC} apply -f kubevirt-hostpath-provisioner-csi/csi-driver/csi-kubevirt-hostpath-provisioner.yaml -n hostpath-provisioner
fi
# create StorageClass crc-csi-hostpath-provisioner
retry ${OC} apply -f kubevirt-hostpath-provisioner-csi/csi-sc.yaml
# Apply registry pvc with crc-csi-hostpath-provisioner StorageClass
retry ${OC} apply -f registry_pvc.yaml
# Add registry storage to pvc
retry ${OC} patch config.imageregistry.operator.openshift.io/cluster --patch='[{"op": "add", "path": "/spec/storage/pvc", "value": {"claim": "crc-image-registry-storage"}}]' --type=json
# Remove emptyDir as storage for registry
retry ${OC} patch config.imageregistry.operator.openshift.io/cluster --patch='[{"op": "remove", "path": "/spec/storage/emptyDir"}]' --type=json
}
# This follows https://blog.openshift.com/enabling-openshift-4-clusters-to-stop-and-resume-cluster-vms/
# in order to trigger regeneration of the initial 24h certs the installer created on the cluster
function renew_certificates() {
local vm_prefix
vm_prefix=$(get_vm_prefix ${SNC_PRODUCT_NAME})
shutdown_vm ${vm_prefix}-master-0
# Enable the network time sync and set the clock back to present on host
sudo date -s '1 day'
sudo timedatectl set-ntp on
start_vm ${vm_prefix}-master-0 api.${SNC_PRODUCT_NAME}.${BASE_DOMAIN}
# Loop until the kubelet certs are valid for a month
i=0
while [ $i -lt 60 ]; do
if ! ${SSH} core@api.${SNC_PRODUCT_NAME}.${BASE_DOMAIN} -- sudo openssl x509 -checkend 2160000 -noout -in /var/lib/kubelet/pki/kubelet-client-current.pem ||
! ${SSH} core@api.${SNC_PRODUCT_NAME}.${BASE_DOMAIN} -- sudo openssl x509 -checkend 2160000 -noout -in /var/lib/kubelet/pki/kubelet-server-current.pem; then
retry ${OC} get csr -ojson > certs.json
retry ${OC} adm certificate approve -f certs.json
rm -f certs.json
echo "Retry loop $i, wait for 10sec before starting next loop"
sleep 10
else
break
fi
i=$[$i+1]
done
if ! ${SSH} core@api.${SNC_PRODUCT_NAME}.${BASE_DOMAIN} -- sudo openssl x509 -checkend 2160000 -noout -in /var/lib/kubelet/pki/kubelet-client-current.pem; then
echo "kubelet client certs are not yet rotated to have 30 days validity"
exit 1
fi
if ! ${SSH} core@api.${SNC_PRODUCT_NAME}.${BASE_DOMAIN} -- sudo openssl x509 -checkend 2160000 -noout -in /var/lib/kubelet/pki/kubelet-server-current.pem; then
echo "kubelet server certs are not yet rotated to have 30 days validity"
exit 1
fi
}
# deletes an operator and wait until the resources it manages are gone.
function delete_operator() {
local delete_object=$1
local namespace=$2
local pod_selector=$3
retry ${OC} get pods
pod=$(${OC} get pod -l ${pod_selector} -o jsonpath="{.items[0].metadata.name}" -n ${namespace})
retry ${OC} delete ${delete_object} -n ${namespace}
# Wait until the operator pod is deleted before trying to delete the resources it manages
${OC} wait --for=delete pod/${pod} --timeout=120s -n ${namespace} || ${OC} delete pod/${pod} --grace-period=0 --force -n ${namespace} || true
}
function all_operators_available() {
# Check the cluster operator output, status for available should be true for all operators
# The single line oc get co output should not contain any occurrences of `False`
${OC} get co -ojsonpath='{.items[*].status.conditions[?(@.type=="Available")].status}' | grep -v False
}
function no_operators_progressing() {
# Check the cluster operator output, status for progressing should be false for all operators
# The single line oc get co output should not contain any occurrences of `True`
${OC} get co -ojsonpath='{.items[*].status.conditions[?(@.type=="Progressing")].status}' | grep -v True
}
function no_operators_degraded() {
# Check the cluster operator output, status for available should be false for all operators
# The single line oc get co output should not contain any occurrences of `True`
${OC} get co -ojsonpath='{.items[*].status.conditions[?(@.type=="Degraded")].status}' | grep -v True
}
function all_pods_are_running_completed() {
local ignoreNamespace=$1
! ${OC} get pod --no-headers --all-namespaces --field-selector=metadata.namespace!="${ignoreNamespace}" | grep -v Running | grep -v Completed
}
function wait_till_cluster_stable() {
sleep 1m
local retryCount=30
local numConsecutive=3
local count=0
local ignoreNamespace=${1:-"none"}
# Remove all the failed Pods
retry ${OC} delete pods --field-selector=status.phase=Failed -A
local a=0
while [ $a -lt $retryCount ]; do
if all_operators_available && no_operators_progressing && no_operators_degraded; then
echo "All operators are available. Ensuring stability ..."
count=$((count + 1))
else
echo "Some operators are still not available"
count=0
fi
if [ $count -eq $numConsecutive ]; then
echo "Cluster has stabilized"
retry ${OC} delete pods --field-selector=status.phase=Failed -A
break
fi
sleep 30s
a=$((a + 1))
done
# Wait till all the pods are either running or complete state
retry all_pods_are_running_completed "${ignoreNamespace}"
}