Skip to content

Commit

Permalink
improve some details
Browse files Browse the repository at this point in the history
  • Loading branch information
@windydayc
windydayc committed Aug 10, 2022
1 parent c3c3726 commit 09ae03d
Show file tree
Hide file tree
Showing 7 changed files with 125 additions and 49 deletions.
72 changes: 67 additions & 5 deletions config/yurtadm/cluster-image/README.md
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,75 @@
# Build an OpenYurt ClusterImage
# OpenYurt ClusterImage

`yurtadm init` is implemented by sealer, you can modify the kubefile to make your own openyurt cluster image.
Currently, `yurtadm init` is implemented by sealer v0.8.5 to create kubernetes master nodes.

## install sealer

```bash
wget https://github.com/sealerio/sealer/releases/download/v0.8.5/sealer-v0.8.5-linux-amd64.tar.gz
tar -zxvf sealer-v0.8.5-linux-amd64.tar.gz -C /usr/bin
```

## Build your own OpenYurt Cluster

Modify the Kubefile to build your own OpenYurt cluster image.

### 1. Build OpenYurt Cluster Image

```bash
cd openyurt-latest

# build openyurt ClusterImage
sealer build -t registry-1.docker.io/your_dockerhub_username/openyurt-cluster:latest-k8s-1198 -f Kubefile .
sealer build -t registry-1.docker.io/your_dockerhub_username/openyurt-cluster:latest-k8s-1.19.8 -f Kubefile .

# push to dockerhub
sealer push registry-1.docker.io/your_dockerhub_username/openyurt-cluster:latest-k8s-1198
```
sealer push registry-1.docker.io/your_dockerhub_username/openyurt-cluster:latest-k8s-1.19.8
```

### 2. Make a Clusterfile

A sample Clusterfile:

```yaml
apiVersion: sealer.cloud/v2
kind: Cluster
metadata:
name: my-cluster
spec:
hosts:
- ips: [ 1.2.3.4 ]
roles: [ master ]
image: registry-1.docker.io/your_dockerhub_username/openyurt-cluster:latest-k8s-1.19.8
ssh:
passwd: xxx
pk: /root/.ssh/id_rsa
user: root
env:
- PodCIDR=10.244.0.0/16
---

## Custom configurations must specify kind, will be merged to default kubeadm configs
kind: ClusterConfiguration
networking:
podSubnet: 10.244.0.0/16
serviceSubnet: 10.96.0.0/12
controllerManager:
extraArgs:
controllers: -nodelifecycle,*,bootstrapsigner,tokencleaner

---

## Custom configurations must specify kind
kind: KubeProxyConfiguration
bindAddress: 0.0.0.0
clusterCIDR: 10.244.0.0/16
featuregates:
EndpointSliceProxying: true
```
### 3. Run OpenYurt Cluster
```bash
sealer apply -f Clusterfile
```

Note: `yurtadm init` only creates master nodes. For worker nodes, you should use `yurtadm join`.
3 changes: 3 additions & 0 deletions config/yurtadm/cluster-image/openyurt-latest/Kubefile
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
FROM kubernetes:v1.19.8-alpine

# flannel: https://github.com/sealerio/applications/tree/main/flannel
# cni has been replaced by the content in openyurt-cni-0.8.7-0.x86_64.rpm in https://github.com/openyurtio/openyurt/releases
# Reason see: https://openyurt.io/docs/user-manuals/network/edge-pod-network/#ipam-pod-ip-address-kept

COPY cni .
COPY init-kube.sh /scripts/
COPY kube-flannel.yaml.tmpl manifests/
Expand Down
5 changes: 1 addition & 4 deletions config/yurtadm/cluster-image/openyurt-latest/install.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,6 @@ echo "[INFO] Start installing OpenYurt."

kubectl apply -f manifests/kube-flannel.yaml

## label node
kubectl label node $HOSTNAME openyurt.io/is-edge-worker=false

## install openyurt components
kubectl apply -f manifests/yurt-controller-manager.yaml
kubectl apply -f manifests/yurt-tunnel-agent.yaml
Expand All @@ -34,6 +31,6 @@ kubectl apply -f manifests/coredns.yaml
kubectl annotate svc kube-dns -n kube-system openyurt.io/topologyKeys='openyurt.io/nodepool'

## configure kube-proxy
kubectl patch cm -n kube-system kube-proxy --patch '{"data": {"config.conf": "apiVersion: kubeproxy.config.k8s.io/v1alpha1\nbindAddress: 0.0.0.0\nfeatureGates:\n EndpointSliceProxying: true\nbindAddressHardFail: false\nclusterCIDR: 100.64.0.0/10\nconfigSyncPeriod: 0s\nenableProfiling: false\nipvs:\n excludeCIDRs:\n - 10.103.97.2/32\n minSyncPeriod: 0s\n strictARP: false\nkind: KubeProxyConfiguration\nmode: ipvs\nudpIdleTimeout: 0s\nwinkernel:\n enableDSR: false\nkubeconfig.conf:"}}' && kubectl delete pod --selector k8s-app=kube-proxy -n kube-system
kubectl patch cm kube-proxy -n kube-system --patch '{"data":{"kubeconfig.conf":""}}' && kubectl delete pod -n kube-system --selector k8s-app=kube-proxy

echo "[INFO] OpenYurt is successfully installed."
35 changes: 28 additions & 7 deletions pkg/yurtadm/cmd/yurtinit/init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,17 +43,21 @@ const (
APIServerAdvertiseAddress = "apiserver-advertise-address"
//YurttunnelServerAddress flag sets the IP address of Yurttunnel Server.
YurttunnelServerAddress = "yurt-tunnel-server-address"
// NetworkingServiceSubnet flag sets the range of IP address for service VIPs.
NetworkingServiceSubnet = "service-cidr"
// NetworkingPodSubnet flag sets the range of IP addresses for the pod network. If set, the control plane will automatically allocate CIDRs for every node.
NetworkingPodSubnet = "pod-network-cidr"
// NetworkingServiceSubnet flag sets the subnet used by kubernetes Services.
NetworkingServiceSubnet = "service-subnet"
// NetworkingPodSubnet flag sets the subnet used by Pods.
NetworkingPodSubnet = "pod-subnet"
// ClusterCIDR flag sets the CIDR range of the pods in the cluster. It is used to bridge traffic coming from outside of the cluster.
ClusterCIDR = "cluster-cidr"
// KubeProxyBindAddress flag sets the IP address for the proxy server to serve on (set to 0.0.0.0 for all interfaces)
KubeProxyBindAddress = "kube-proxy-bind-address"
// OpenYurtVersion flag sets the OpenYurt version for the control plane.
OpenYurtVersion = "openyurt-version"
// K8sVersion flag sets the Kubernetes version for the control plane.
K8sVersion = "k8s-version"
// ImageRepository flag sets the container registry to pull control plane images from.
ImageRepository = "image-repository"
// PassWd flag is the password of master server.
// PassWd flag sets the password of master server.
PassWd = "passwd"

TmpDownloadDir = "/tmp"
Expand Down Expand Up @@ -91,6 +95,15 @@ networking:
controllerManager:
extraArgs:
controllers: -nodelifecycle,*,bootstrapsigner,tokencleaner
---
## Custom configurations must specify kind
kind: KubeProxyConfiguration
bindAddress: {{.bind_address}}
clusterCIDR: {{.cluster_cidr}}
featuregates:
EndpointSliceProxying: true
`
)

Expand Down Expand Up @@ -150,11 +163,11 @@ func addFlags(flagset *flag.FlagSet, o *InitOptions) {
"The yurt-tunnel-server address.")
flagset.StringVarP(
&o.ServiceSubnet, NetworkingServiceSubnet, "", o.ServiceSubnet,
"Use alternative range of IP address for service VIPs.",
"ServiceSubnet is the subnet used by kubernetes Services.",
)
flagset.StringVarP(
&o.PodSubnet, NetworkingPodSubnet, "", o.PodSubnet,
"Specify range of IP addresses for the pod network. If set, the control plane will automatically allocate CIDRs for every node.",
"PodSubnet is the subnet used by Pods.",
)
flagset.StringVarP(&o.Password, PassWd, "p", o.Password,
"set master server ssh password",
Expand All @@ -170,6 +183,12 @@ func addFlags(flagset *flag.FlagSet, o *InitOptions) {
flagset.StringVarP(&o.ImageRepository, ImageRepository, "", o.ImageRepository,
"Choose a registry to pull cluster images from",
)
flagset.StringVarP(&o.ClusterCIDR, ClusterCIDR, "", o.ClusterCIDR,
"Choose a CIDR range of the pods in the cluster",
)
flagset.StringVarP(&o.KubeProxyBindAddress, KubeProxyBindAddress, "", o.KubeProxyBindAddress,
"Choose an IP address for the proxy server to serve on",
)
}

func NewInitializerWithOptions(o *InitOptions) *clusterInitializer {
Expand Down Expand Up @@ -257,6 +276,8 @@ func (ci *clusterInitializer) PrepareClusterfile() error {
"pod_subnet": ci.PodSubnet,
"service_subnet": ci.ServiceSubnet,
"yurttunnel_server_address": ci.YurttunnelServerAddress,
"cluster_cidr": ci.ClusterCIDR,
"bind_address": ci.KubeProxyBindAddress,
})
if err != nil {
return err
Expand Down
14 changes: 9 additions & 5 deletions pkg/yurtadm/cmd/yurtinit/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,19 +32,23 @@ type InitOptions struct {
YurttunnelServerAddress string
ServiceSubnet string
PodSubnet string
ClusterCIDR string
Password string
ImageRepository string
OpenYurtVersion string
K8sVersion string
KubeProxyBindAddress string
}

func NewInitOptions() *InitOptions {
return &InitOptions{
ImageRepository: constants.DefaultOpenYurtImageRegistry,
OpenYurtVersion: constants.DefaultOpenYurtVersion,
K8sVersion: constants.DefaultK8sVersion,
PodSubnet: constants.DefaultPodSubnet,
ServiceSubnet: constants.DefaultServiceSubnet,
ImageRepository: constants.DefaultOpenYurtImageRegistry,
OpenYurtVersion: constants.DefaultOpenYurtVersion,
K8sVersion: constants.DefaultK8sVersion,
PodSubnet: constants.DefaultPodSubnet,
ServiceSubnet: constants.DefaultServiceSubnet,
ClusterCIDR: constants.DefaultClusterCIDR,
KubeProxyBindAddress: constants.DefaultKubeProxyBindAddress,
}
}

Expand Down
24 changes: 14 additions & 10 deletions pkg/yurtadm/constants/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ const (
YurthubStaticPodFileName = "yurthub.yaml"
PauseImagePath = "registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.2"

OpenYurtCniUrl = "https://github.com/openyurtio/openyurt/releases/download/v0.7.0/openyurt-cni-0.8.7-0.x86_64.rpm"
CniUrlFormat = "https://aliacs-edge-k8s-cn-hangzhou.oss-cn-hangzhou.aliyuncs.com/public/pkg/openyurt/cni/%s/cni-plugins-linux-%s-%s.tgz"
DefaultKubernetesResourceServer = "dl.k8s.io"
KubeUrlFormat = "https://%s/%s/kubernetes-node-linux-%s.tar.gz"
Expand All @@ -48,18 +49,21 @@ const (

DefaultOpenYurtImageRegistry = "registry.cn-hangzhou.aliyuncs.com/openyurt"
DefaultOpenYurtVersion = "latest"
DefaultK8sVersion = "1198" // default kubernetes 1.19.8
DefaultK8sVersion = "1.19.8" // default kubernetes 1.19.8
DefaultPodSubnet = "10.244.0.0/16"
DefaultServiceSubnet = "10.96.0.0/12"
YurtControllerManager = "yurt-controller-manager"
YurtTunnelServer = "yurt-tunnel-server"
YurtTunnelAgent = "yurt-tunnel-agent"
Yurthub = "yurthub"
YurtAppManager = "yurt-app-manager"
YurtAppManagerNamespace = "kube-system"
DirMode = 0755
FileMode = 0666
KubeletServiceContent = `
DefaultClusterCIDR = "10.244.0.0/16"
DefaultKubeProxyBindAddress = "0.0.0.0"

YurtControllerManager = "yurt-controller-manager"
YurtTunnelServer = "yurt-tunnel-server"
YurtTunnelAgent = "yurt-tunnel-agent"
Yurthub = "yurthub"
YurtAppManager = "yurt-app-manager"
YurtAppManagerNamespace = "kube-system"
DirMode = 0755
FileMode = 0666
KubeletServiceContent = `
[Unit]
Description=kubelet: The Kubernetes Node Agent
Documentation=http://kubernetes.io/docs/
Expand Down
21 changes: 3 additions & 18 deletions pkg/yurtadm/util/kubernetes/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ func CheckAndInstallKubelet(kubernetesResourceServer, clusterVersion string) err
savePath := fmt.Sprintf("%s/kubernetes-node-linux-%s.tar.gz", constants.TmpDownloadDir, runtime.GOARCH)
klog.V(1).Infof("Download kubelet from: %s", packageUrl)
if err := util.DownloadFile(packageUrl, savePath, 3); err != nil {
return fmt.Errorf("Download kuelet fail: %w", err)
return fmt.Errorf("Download kubelet fail: %w", err)
}
if err := util.Untar(savePath, constants.TmpDownloadDir); err != nil {
return err
Expand All @@ -145,23 +145,8 @@ func CheckAndInstallKubelet(kubernetesResourceServer, clusterVersion string) err
return nil
}

////download and install kubernetes-cni
//cniUrl := fmt.Sprintf(constants.CniUrlFormat, constants.KubeCniVersion, runtime.GOARCH, constants.KubeCniVersion)
//savePath := fmt.Sprintf("%s/cni-plugins-linux-%s-%s.tgz", constants.TmpDownloadDir, runtime.GOARCH, constants.KubeCniVersion)
//klog.V(1).Infof("Download cni from: %s", cniUrl)
//if err := util.DownloadFile(cniUrl, savePath, 3); err != nil {
// return err
//}
//
//if err := os.MkdirAll(constants.KubeCniDir, 0600); err != nil {
// return err
//}
//if err := util.Untar(savePath, constants.KubeCniDir); err != nil {
// return err
//}

//download and install kubernetes-cni
cniUrl := "https://github.com/openyurtio/openyurt/releases/download/v0.7.0/openyurt-cni-0.8.7-0.x86_64.rpm"
// download and install openyurt-cni
cniUrl := fmt.Sprintf("%s", constants.OpenYurtCniUrl)
savePath := fmt.Sprintf("%s/openyurt-cni-0.8.7-0.x86_64.rpm", constants.TmpDownloadDir)
klog.V(1).Infof("Download cni from: %s", cniUrl)
if err := util.DownloadFile(cniUrl, savePath, 3); err != nil {
Expand Down

0 comments on commit 09ae03d

Please sign in to comment.