Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: DNS configuration exception #574

Merged
merged 1 commit into from
May 22, 2024
Merged

fix: DNS configuration exception #574

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions deploy/virtual-cluster-host-port-cm.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
---
apiVersion: v1
kind: ConfigMap
metadata:
name: kosmos-hostports
namespace: kosmos-system
data:
config.yaml: |
# ports allocate for virtual cluster api server,from 33001, increment by 1 for each virtual cluster.Be careful not to use ports that are already in use
portsPool:
- 33001
- 33002
- 33003
- 33004
- 33005
- 33006
- 33007
- 33008
- 33009
- 33010
# when port is allocate from pool,it will be used for virtual cluster api server,and the port will be released after virtual cluster is deleted
clusterPorts:
315 changes: 15 additions & 300 deletions deploy/virtual-cluster-operator.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,287 +51,15 @@ metadata:
name: virtual-cluster-operator
namespace: kosmos-system
data:
# Generate by script hack/k8s-in-k8s/generate_env.sh
env.sh: |
#!/usr/bin/env bash

SCRIPT_VERSION=0.0.1
# save tmp file
PATH_FILE_TMP=/apps/conf/kosmos/tmp
###################################################
# path for kubeadm
PATH_KUBEADM=/usr/bin/kubeadm
##################################################
# path for kubeadm config
PATH_KUBEADM_CONFIG=/etc/kubeadm
##################################################
# path for kubernetes
PATH_KUBERNETES=/etc/kubernetes/
PATH_KUBERNETES_PKI="$PATH_KUBERNETES/pki"
# scpKCCmd.name
KUBELET_KUBE_CONFIG_NAME=kubelet.conf
##################################################
# path for kubelet
PATH_KUBELET_LIB=/var/lib/kubelet
# scpKubeletConfigCmd.name
KUBELET_CONFIG_NAME=config.yaml
__env.sh__
# Copy from hack/k8s-in-k8s/kubelet_node_helper.sh
kubelet_node_helper.sh: |
#!/usr/bin/env bash

source "env.sh"

# args
DNS_ADDRESS=${2:-10.237.0.10}
LOG_NAME=${2:-kubelet}
JOIN_HOST=$2
JOIN_TOKEN=$3
JOIN_CA_HASH=$4

function unjoin() {
# before unjoin, you need delete node by kubectl
echo "exec(1/2): kubeadm reset...."
echo "y" | ${PATH_KUBEADM} reset
if [ $? -ne 0 ]; then
exit 1
fi

echo "exec(2/2): delete cni...."
if [ -d "/etc/cni/net.d" ]; then
mv /etc/cni/net.d '/etc/cni/net.d.back'`date +%Y_%m_%d_%H_%M_%S`
if [ $? -ne 0 ]; then
exit 1
fi
fi
}

function revert() {
if [ ! -f "$PATH_KUBEADM_CONFIG/kubeadm.cfg" ]; then
echo "exec(1/1): execure join cmd"
kubeadm join $JOIN_HOST --token $JOIN_TOKEN --discovery-token-ca-cert-hash $JOIN_CA_HASH
if [ $? -ne 0 ]; then
exit 1
fi
exit 0
fi

echo "exec(1/3): update kubeadm.cfg..."
sed -e "s|token: .*$|token: $JOIN_TOKEN|g" -e "w $PATH_FILE_TMP/kubeadm.cfg.current" "$PATH_KUBEADM_CONFIG/kubeadm.cfg"
if [ $? -ne 0 ]; then
exit 1
fi

# add taints
echo "exec(2/3): update kubeadm.cfg tanits..."
sed -i "/kubeletExtraArgs/a \ register-with-taints: node.kosmos.io/unschedulable:NoSchedule" "$PATH_FILE_TMP/kubeadm.cfg.current"
if [ $? -ne 0 ]; then
exit 1
fi

echo "exec(3/3): execute join cmd...."
kubeadm join --config "$PATH_FILE_TMP/kubeadm.cfg.current"
if [ $? -ne 0 ]; then
exit 1
fi
}

# before join, you need upload ca.crt and kubeconfig to tmp dir!!!
function join() {
echo "exec(1/8): stop containerd...."
systemctl stop containerd
if [ $? -ne 0 ]; then
exit 1
fi
echo "exec(2/8): copy ca.crt...."
cp "$PATH_FILE_TMP/ca.crt" "$PATH_KUBERNETES_PKI/ca.crt"
if [ $? -ne 0 ]; then
exit 1
fi
echo "exec(3/8): copy kubeconfig...."
cp "$PATH_FILE_TMP/$KUBELET_KUBE_CONFIG_NAME" "$PATH_KUBERNETES/$KUBELET_KUBE_CONFIG_NAME"
if [ $? -ne 0 ]; then
exit 1
fi
echo "exec(4/8): set core dns address...."
sed -e "s|__DNS_ADDRESS__|$DNS_ADDRESS|g" -e "w ${PATH_KUBELET_LIB}/${KUBELET_CONFIG_NAME}" "$PATH_FILE_TMP"/"$KUBELET_CONFIG_NAME"
if [ $? -ne 0 ]; then
exit 1
fi
echo "exec(5/8): copy kubeadm-flags.env...."
cp "$PATH_FILE_TMP/kubeadm-flags.env" "$PATH_KUBELET_LIB/kubeadm-flags.env"
if [ $? -ne 0 ]; then
exit 1
fi
echo "exec(6/8): start containerd"
systemctl start containerd
if [ $? -ne 0 ]; then
exit 1
fi

echo "exec(7/8): delete cni...."
if [ -d "/etc/cni/net.d" ]; then
mv /etc/cni/net.d '/etc/cni/net.d.back'`date +%Y_%m_%d_%H_%M_%S`
if [ $? -ne 0 ]; then
exit 1
fi
fi

echo "exec(8/8): start kubelet...."
systemctl start kubelet
if [ $? -ne 0 ]; then
exit 1
fi
}

function health() {
result=`systemctl is-active containerd`
if [[ $result != "active" ]]; then
echo "health(1/2): containerd is inactive"
exit 1
else
echo "health(1/2): containerd is active"
fi

result=`systemctl is-active kubelet`
if [[ $result != "active" ]]; then
echo "health(2/2): kubelet is inactive"
exit 1
else
echo "health(2/2): containerd is active"
fi
}

function log() {
systemctl status $LOG_NAME
}

# check the environments
function check() {
echo "check(1/3): try to create $PATH_FILE_TMP"
if [ ! -d "$PATH_FILE_TMP" ]; then
mkdir -p "$PATH_FILE_TMP"
if [ $? -ne 0 ]; then
exit 1
fi
fi

echo "check(2/3): check dir: $PATH_KUBEADM_CONFIG"
if [ ! -d "$PATH_KUBEADM_CONFIG" ]; then
mkdir -p "$PATH_KUBEADM_CONFIG"
if [ $? -ne 0 ]; then
exit 1
fi

echo "---
apiVersion: kubeadm.k8s.io/v1beta2
discovery:
bootstrapToken:
apiServerEndpoint: apiserver.cluster.local:6443
token: xxxxxxxx
unsafeSkipCAVerification: true
kind: JoinConfiguration
nodeRegistration:
criSocket: /run/containerd/containerd.sock
kubeletExtraArgs:
container-runtime: remote
container-runtime-endpoint: unix:///run/containerd/containerd.sock
taints: null" > $PATH_KUBEADM_CONFIG/kubeadm.cfg

fi

echo "check(3/3): copy kubeadm-flags.env to create $PATH_FILE_TMP , remove args[cloud-provider] and taints"
sed -e "s| --cloud-provider=external | |g" -e "w ${PATH_FILE_TMP}/kubeadm-flags.env" "$PATH_KUBELET_LIB/kubeadm-flags.env"
sed -i "s| --register-with-taints=node.kosmos.io/unschedulable:NoSchedule||g" "${PATH_FILE_TMP}/kubeadm-flags.env"
if [ $? -ne 0 ]; then
exit 1
fi

echo "environments is ok"
}

function version() {
echo "$SCRIPT_VERSION"
}

# See how we were called.
case "$1" in
unjoin)
unjoin
;;
join)
join
;;
health)
health
;;
check)
check
;;
log)
log
;;
revert)
revert
;;
version)
version
;;
*)
echo $"usage: $0 unjoin|join|health|log|check|version|revert"
exit 1
esac
__kubelet_node_helper__
# Obtain through the command "kubectl get cm kubelet-config -nkube-system -oyaml", change dns address to `__DNS_ADDRESS__`
config.yaml: |
apiVersion: kubelet.config.k8s.io/v1beta1
authentication:
anonymous:
enabled: false
webhook:
cacheTTL: 0s
enabled: true
x509:
clientCAFile: /etc/kubernetes/pki/ca.crt
authorization:
mode: Webhook
webhook:
cacheAuthorizedTTL: 0s
cacheUnauthorizedTTL: 0s
cgroupDriver: cgroupfs
clusterDNS:
- __DNS_ADDRESS__
clusterDomain: cluster.local
cpuManagerReconcilePeriod: 0s
evictionHard:
imagefs.available: 15%
memory.available: 100Mi
nodefs.available: 10%
nodefs.inodesFree: 5%
evictionPressureTransitionPeriod: 5m0s
fileCheckFrequency: 0s
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 0s
imageMinimumGCAge: 0s
kind: KubeletConfiguration
kubeAPIBurst: 100
kubeAPIQPS: 100
kubeReserved:
cpu: 140m
memory: 1.80G
logging:
flushFrequency: 0
options:
json:
infoBufferSize: "0"
verbosity: 0
memorySwap: {}
nodeStatusReportFrequency: 0s
nodeStatusUpdateFrequency: 0s
rotateCertificates: true
runtimeRequestTimeout: 0s
shutdownGracePeriod: 0s
shutdownGracePeriodCriticalPods: 0s
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 0s
syncFrequency: 0s
volumeStatsAggPeriod: 0s
__config.yaml__
---
apiVersion: apps/v1
kind: Deployment
Expand Down Expand Up @@ -388,6 +116,12 @@ spec:
# Enter the ip address of a master node
- name: EXECTOR_HOST_MASTER_NODE_IP
value: 192.168.0.1
# env.sh KUBELET_CONFIG_NAME
- name: KUBELET_CONFIG_NAME
value: config.yaml
# env.sh KUBELET_KUBE_CONFIG_NAME
- name: KUBELET_KUBE_CONFIG_NAME
value: kubelet.conf
# WEB_USER and WEB_PASS for generate token that can be used to access the node-agent
- name: WEB_USER
valueFrom:
Expand All @@ -399,6 +133,8 @@ spec:
secretKeyRef:
name: node-agent-secret
key: password
- name: EXECTOR_SHELL_NAME
value: kubelet_node_helper.sh
volumeMounts:
- name: credentials
mountPath: /etc/virtual-cluster-operator
Expand Down Expand Up @@ -513,25 +249,4 @@ type: kubernetes.io/basic-auth
data:
username: {{ .USERNAME }}
password: {{ .PASSWORD }}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: kosmos-hostports
namespace: kosmos-system
data:
config.yaml: |
# ports allocate for virtual cluster api server,from 33001, increment by 1 for each virtual cluster.Be careful not to use ports that are already in use
portsPool:
- 33001
- 33002
- 33003
- 33004
- 33005
- 33006
- 33007
- 33008
- 33009
- 33010
# when port is allocate from pool,it will be used for virtual cluster api server,and the port will be released after virtual cluster is deleted
clusterPorts:

22 changes: 0 additions & 22 deletions hack/k8s-in-k8s/env.dev.sh
View file
Open in desktop

This file was deleted.

Loading
Loading