Skip to content

Commit

Permalink
fix: DNS configuration exception
Browse files Browse the repository at this point in the history 
Signed-off-by: OrangeBao <baoyinghai_yewu@cmss.chinamobile.com>
  • Loading branch information
@OrangeBao
OrangeBao committed May 21, 2024
1 parent f4be68e commit 8db228b
Show file tree
Hide file tree
Showing 11 changed files with 309 additions and 419 deletions.
22 changes: 22 additions & 0 deletions deploy/virtual-cluster-host-port-cm.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
---
apiVersion: v1
kind: ConfigMap
metadata:
name: kosmos-hostports
namespace: kosmos-system
data:
config.yaml: |
# ports allocate for virtual cluster api server,from 33001, increment by 1 for each virtual cluster.Be careful not to use ports that are already in use
portsPool:
- 33001
- 33002
- 33003
- 33004
- 33005
- 33006
- 33007
- 33008
- 33009
- 33010
# when port is allocate from pool,it will be used for virtual cluster api server,and the port will be released after virtual cluster is deleted
clusterPorts:
315 changes: 15 additions & 300 deletions deploy/virtual-cluster-operator.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,287 +51,15 @@ metadata:
name: virtual-cluster-operator
namespace: kosmos-system
data:
# Generate by script hack/k8s-in-k8s/generate_env.sh
env.sh: |
#!/usr/bin/env bash
SCRIPT_VERSION=0.0.1
# save tmp file
PATH_FILE_TMP=/apps/conf/kosmos/tmp
###################################################
# path for kubeadm
PATH_KUBEADM=/usr/bin/kubeadm
##################################################
# path for kubeadm config
PATH_KUBEADM_CONFIG=/etc/kubeadm
##################################################
# path for kubernetes
PATH_KUBERNETES=/etc/kubernetes/
PATH_KUBERNETES_PKI="$PATH_KUBERNETES/pki"
# scpKCCmd.name
KUBELET_KUBE_CONFIG_NAME=kubelet.conf
##################################################
# path for kubelet
PATH_KUBELET_LIB=/var/lib/kubelet
# scpKubeletConfigCmd.name
KUBELET_CONFIG_NAME=config.yaml
__env.sh__
# Copy from hack/k8s-in-k8s/kubelet_node_helper.sh
kubelet_node_helper.sh: |
#!/usr/bin/env bash
source "env.sh"
# args
DNS_ADDRESS=${2:-10.237.0.10}
LOG_NAME=${2:-kubelet}
JOIN_HOST=$2
JOIN_TOKEN=$3
JOIN_CA_HASH=$4
function unjoin() {
# before unjoin, you need delete node by kubectl
echo "exec(1/2): kubeadm reset...."
echo "y" | ${PATH_KUBEADM} reset
if [ $? -ne 0 ]; then
exit 1
fi
echo "exec(2/2): delete cni...."
if [ -d "/etc/cni/net.d" ]; then
mv /etc/cni/net.d '/etc/cni/net.d.back'`date +%Y_%m_%d_%H_%M_%S`
if [ $? -ne 0 ]; then
exit 1
fi
fi
}
function revert() {
if [ ! -f "$PATH_KUBEADM_CONFIG/kubeadm.cfg" ]; then
echo "exec(1/1): execure join cmd"
kubeadm join $JOIN_HOST --token $JOIN_TOKEN --discovery-token-ca-cert-hash $JOIN_CA_HASH
if [ $? -ne 0 ]; then
exit 1
fi
exit 0
fi
echo "exec(1/3): update kubeadm.cfg..."
sed -e "s|token: .*$|token: $JOIN_TOKEN|g" -e "w $PATH_FILE_TMP/kubeadm.cfg.current" "$PATH_KUBEADM_CONFIG/kubeadm.cfg"
if [ $? -ne 0 ]; then
exit 1
fi
# add taints
echo "exec(2/3): update kubeadm.cfg tanits..."
sed -i "/kubeletExtraArgs/a \ register-with-taints: node.kosmos.io/unschedulable:NoSchedule" "$PATH_FILE_TMP/kubeadm.cfg.current"
if [ $? -ne 0 ]; then
exit 1
fi
echo "exec(3/3): execute join cmd...."
kubeadm join --config "$PATH_FILE_TMP/kubeadm.cfg.current"
if [ $? -ne 0 ]; then
exit 1
fi
}
# before join, you need upload ca.crt and kubeconfig to tmp dir!!!
function join() {
echo "exec(1/8): stop containerd...."
systemctl stop containerd
if [ $? -ne 0 ]; then
exit 1
fi
echo "exec(2/8): copy ca.crt...."
cp "$PATH_FILE_TMP/ca.crt" "$PATH_KUBERNETES_PKI/ca.crt"
if [ $? -ne 0 ]; then
exit 1
fi
echo "exec(3/8): copy kubeconfig...."
cp "$PATH_FILE_TMP/$KUBELET_KUBE_CONFIG_NAME" "$PATH_KUBERNETES/$KUBELET_KUBE_CONFIG_NAME"
if [ $? -ne 0 ]; then
exit 1
fi
echo "exec(4/8): set core dns address...."
sed -e "s|__DNS_ADDRESS__|$DNS_ADDRESS|g" -e "w ${PATH_KUBELET_LIB}/${KUBELET_CONFIG_NAME}" "$PATH_FILE_TMP"/"$KUBELET_CONFIG_NAME"
if [ $? -ne 0 ]; then
exit 1
fi
echo "exec(5/8): copy kubeadm-flags.env...."
cp "$PATH_FILE_TMP/kubeadm-flags.env" "$PATH_KUBELET_LIB/kubeadm-flags.env"
if [ $? -ne 0 ]; then
exit 1
fi
echo "exec(6/8): start containerd"
systemctl start containerd
if [ $? -ne 0 ]; then
exit 1
fi
echo "exec(7/8): delete cni...."
if [ -d "/etc/cni/net.d" ]; then
mv /etc/cni/net.d '/etc/cni/net.d.back'`date +%Y_%m_%d_%H_%M_%S`
if [ $? -ne 0 ]; then
exit 1
fi
fi
echo "exec(8/8): start kubelet...."
systemctl start kubelet
if [ $? -ne 0 ]; then
exit 1
fi
}
function health() {
result=`systemctl is-active containerd`
if [[ $result != "active" ]]; then
echo "health(1/2): containerd is inactive"
exit 1
else
echo "health(1/2): containerd is active"
fi
result=`systemctl is-active kubelet`
if [[ $result != "active" ]]; then
echo "health(2/2): kubelet is inactive"
exit 1
else
echo "health(2/2): containerd is active"
fi
}
function log() {
systemctl status $LOG_NAME
}
# check the environments
function check() {
echo "check(1/3): try to create $PATH_FILE_TMP"
if [ ! -d "$PATH_FILE_TMP" ]; then
mkdir -p "$PATH_FILE_TMP"
if [ $? -ne 0 ]; then
exit 1
fi
fi
echo "check(2/3): check dir: $PATH_KUBEADM_CONFIG"
if [ ! -d "$PATH_KUBEADM_CONFIG" ]; then
mkdir -p "$PATH_KUBEADM_CONFIG"
if [ $? -ne 0 ]; then
exit 1
fi
echo "---
apiVersion: kubeadm.k8s.io/v1beta2
discovery:
bootstrapToken:
apiServerEndpoint: apiserver.cluster.local:6443
token: xxxxxxxx
unsafeSkipCAVerification: true
kind: JoinConfiguration
nodeRegistration:
criSocket: /run/containerd/containerd.sock
kubeletExtraArgs:
container-runtime: remote
container-runtime-endpoint: unix:///run/containerd/containerd.sock
taints: null" > $PATH_KUBEADM_CONFIG/kubeadm.cfg
fi
echo "check(3/3): copy kubeadm-flags.env to create $PATH_FILE_TMP , remove args[cloud-provider] and taints"
sed -e "s| --cloud-provider=external | |g" -e "w ${PATH_FILE_TMP}/kubeadm-flags.env" "$PATH_KUBELET_LIB/kubeadm-flags.env"
sed -i "s| --register-with-taints=node.kosmos.io/unschedulable:NoSchedule||g" "${PATH_FILE_TMP}/kubeadm-flags.env"
if [ $? -ne 0 ]; then
exit 1
fi
echo "environments is ok"
}
function version() {
echo "$SCRIPT_VERSION"
}
# See how we were called.
case "$1" in
unjoin)
unjoin
;;
join)
join
;;
health)
health
;;
check)
check
;;
log)
log
;;
revert)
revert
;;
version)
version
;;
*)
echo $"usage: $0 unjoin|join|health|log|check|version|revert"
exit 1
esac
__kubelet_node_helper__
# Obtain through the command "kubectl get cm kubelet-config -nkube-system -oyaml", change dns address to `__DNS_ADDRESS__`
config.yaml: |
apiVersion: kubelet.config.k8s.io/v1beta1
authentication:
anonymous:
enabled: false
webhook:
cacheTTL: 0s
enabled: true
x509:
clientCAFile: /etc/kubernetes/pki/ca.crt
authorization:
mode: Webhook
webhook:
cacheAuthorizedTTL: 0s
cacheUnauthorizedTTL: 0s
cgroupDriver: cgroupfs
clusterDNS:
- __DNS_ADDRESS__
clusterDomain: cluster.local
cpuManagerReconcilePeriod: 0s
evictionHard:
imagefs.available: 15%
memory.available: 100Mi
nodefs.available: 10%
nodefs.inodesFree: 5%
evictionPressureTransitionPeriod: 5m0s
fileCheckFrequency: 0s
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 0s
imageMinimumGCAge: 0s
kind: KubeletConfiguration
kubeAPIBurst: 100
kubeAPIQPS: 100
kubeReserved:
cpu: 140m
memory: 1.80G
logging:
flushFrequency: 0
options:
json:
infoBufferSize: "0"
verbosity: 0
memorySwap: {}
nodeStatusReportFrequency: 0s
nodeStatusUpdateFrequency: 0s
rotateCertificates: true
runtimeRequestTimeout: 0s
shutdownGracePeriod: 0s
shutdownGracePeriodCriticalPods: 0s
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 0s
syncFrequency: 0s
volumeStatsAggPeriod: 0s
__config.yaml__
---
apiVersion: apps/v1
kind: Deployment
Expand Down Expand Up @@ -388,6 +116,12 @@ spec:
# Enter the ip address of a master node
- name: EXECTOR_HOST_MASTER_NODE_IP
value: 192.168.0.1
# env.sh KUBELET_CONFIG_NAME
- name: KUBELET_CONFIG_NAME
value: config.yaml
# env.sh KUBELET_KUBE_CONFIG_NAME
- name: KUBELET_KUBE_CONFIG_NAME
value: kubelet.conf
# WEB_USER and WEB_PASS for generate token that can be used to access the node-agent
- name: WEB_USER
valueFrom:
Expand All @@ -399,6 +133,8 @@ spec:
secretKeyRef:
name: node-agent-secret
key: password
- name: EXECTOR_SHELL_NAME
value: kubelet_node_helper.sh
volumeMounts:
- name: credentials
mountPath: /etc/virtual-cluster-operator
Expand Down Expand Up @@ -513,25 +249,4 @@ type: kubernetes.io/basic-auth
data:
username: {{ .USERNAME }}
password: {{ .PASSWORD }}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: kosmos-hostports
namespace: kosmos-system
data:
config.yaml: |
# ports allocate for virtual cluster api server,from 33001, increment by 1 for each virtual cluster.Be careful not to use ports that are already in use
portsPool:
- 33001
- 33002
- 33003
- 33004
- 33005
- 33006
- 33007
- 33008
- 33009
- 33010
# when port is allocate from pool,it will be used for virtual cluster api server,and the port will be released after virtual cluster is deleted
clusterPorts:

22 changes: 0 additions & 22 deletions hack/k8s-in-k8s/env.dev.sh
View file

This file was deleted.

Loading

0 comments on commit 8db228b

Please sign in to comment.