diff --git a/config/yurtadm/cluster-image/README.md b/config/yurtadm/cluster-image/README.md index 23cc0822b7f..25c1c9ce1ec 100644 --- a/config/yurtadm/cluster-image/README.md +++ b/config/yurtadm/cluster-image/README.md @@ -1,13 +1,75 @@ -# Build an OpenYurt ClusterImage +# OpenYurt ClusterImage -`yurtadm init` is implemented by sealer, you can modify the kubefile to make your own openyurt cluster image. +Currently, `yurtadm init` is implemented by sealer v0.8.5 to create kubernetes master nodes. + +## install sealer + +```bash +wget https://github.com/sealerio/sealer/releases/download/v0.8.5/sealer-v0.8.5-linux-amd64.tar.gz +tar -zxvf sealer-v0.8.5-linux-amd64.tar.gz -C /usr/bin +``` + +## Build your own OpenYurt Cluster + +Modify the Kubefile to build your own OpenYurt cluster image. + +### 1. Build OpenYurt Cluster Image ```bash cd openyurt-latest # build openyurt ClusterImage -sealer build -t registry-1.docker.io/your_dockerhub_username/openyurt-cluster:latest-k8s-1198 -f Kubefile . +sealer build -t registry-1.docker.io/your_dockerhub_username/openyurt-cluster:latest-k8s-1.19.8 -f Kubefile . # push to dockerhub -sealer push registry-1.docker.io/your_dockerhub_username/openyurt-cluster:latest-k8s-1198 -``` \ No newline at end of file +sealer push registry-1.docker.io/your_dockerhub_username/openyurt-cluster:latest-k8s-1.19.8 +``` + +### 2. Make a Clusterfile + +A sample Clusterfile: + +```yaml +apiVersion: sealer.cloud/v2 +kind: Cluster +metadata: + name: my-cluster +spec: + hosts: + - ips: [ 1.2.3.4 ] + roles: [ master ] + image: registry-1.docker.io/your_dockerhub_username/openyurt-cluster:latest-k8s-1.19.8 + ssh: + passwd: xxx + pk: /root/.ssh/id_rsa + user: root + env: + - PodCIDR=10.244.0.0/16 +--- + +## Custom configurations must specify kind, will be merged to default kubeadm configs +kind: ClusterConfiguration +networking: + podSubnet: 10.244.0.0/16 + serviceSubnet: 10.96.0.0/12 +controllerManager: + extraArgs: + controllers: -nodelifecycle,*,bootstrapsigner,tokencleaner + +--- + +## Custom configurations must specify kind +kind: KubeProxyConfiguration +bindAddress: 0.0.0.0 +clusterCIDR: 10.244.0.0/16 +featuregates: + EndpointSliceProxying: true +``` + +### 3. Run OpenYurt Cluster + +```bash +sealer apply -f Clusterfile +``` + +Note: `yurtadm init` only creates master nodes. For worker nodes, you should use `yurtadm join`. \ No newline at end of file diff --git a/config/yurtadm/cluster-image/openyurt-latest/Kubefile b/config/yurtadm/cluster-image/openyurt-latest/Kubefile index 6076cb2cd88..666bdeee2b7 100644 --- a/config/yurtadm/cluster-image/openyurt-latest/Kubefile +++ b/config/yurtadm/cluster-image/openyurt-latest/Kubefile @@ -1,6 +1,9 @@ FROM kubernetes:v1.19.8-alpine # flannel: https://github.com/sealerio/applications/tree/main/flannel +# cni has been replaced by the content in openyurt-cni-0.8.7-0.x86_64.rpm in https://github.com/openyurtio/openyurt/releases +# Reason see: https://openyurt.io/docs/user-manuals/network/edge-pod-network/#ipam-pod-ip-address-kept + COPY cni . COPY init-kube.sh /scripts/ COPY kube-flannel.yaml.tmpl manifests/ diff --git a/config/yurtadm/cluster-image/openyurt-latest/install.sh b/config/yurtadm/cluster-image/openyurt-latest/install.sh index 9914fd09f00..ba53a7695ef 100644 --- a/config/yurtadm/cluster-image/openyurt-latest/install.sh +++ b/config/yurtadm/cluster-image/openyurt-latest/install.sh @@ -18,9 +18,6 @@ echo "[INFO] Start installing OpenYurt." kubectl apply -f manifests/kube-flannel.yaml -## label node -kubectl label node $HOSTNAME openyurt.io/is-edge-worker=false - ## install openyurt components kubectl apply -f manifests/yurt-controller-manager.yaml kubectl apply -f manifests/yurt-tunnel-agent.yaml @@ -34,6 +31,6 @@ kubectl apply -f manifests/coredns.yaml kubectl annotate svc kube-dns -n kube-system openyurt.io/topologyKeys='openyurt.io/nodepool' ## configure kube-proxy -kubectl patch cm -n kube-system kube-proxy --patch '{"data": {"config.conf": "apiVersion: kubeproxy.config.k8s.io/v1alpha1\nbindAddress: 0.0.0.0\nfeatureGates:\n EndpointSliceProxying: true\nbindAddressHardFail: false\nclusterCIDR: 100.64.0.0/10\nconfigSyncPeriod: 0s\nenableProfiling: false\nipvs:\n excludeCIDRs:\n - 10.103.97.2/32\n minSyncPeriod: 0s\n strictARP: false\nkind: KubeProxyConfiguration\nmode: ipvs\nudpIdleTimeout: 0s\nwinkernel:\n enableDSR: false\nkubeconfig.conf:"}}' && kubectl delete pod --selector k8s-app=kube-proxy -n kube-system +kubectl patch cm kube-proxy -n kube-system --patch '{"data":{"kubeconfig.conf":""}}' && kubectl delete pod -n kube-system --selector k8s-app=kube-proxy echo "[INFO] OpenYurt is successfully installed." \ No newline at end of file diff --git a/pkg/yurtadm/cmd/yurtinit/init.go b/pkg/yurtadm/cmd/yurtinit/init.go index 03387d8b493..db56b57748c 100644 --- a/pkg/yurtadm/cmd/yurtinit/init.go +++ b/pkg/yurtadm/cmd/yurtinit/init.go @@ -43,17 +43,21 @@ const ( APIServerAdvertiseAddress = "apiserver-advertise-address" //YurttunnelServerAddress flag sets the IP address of Yurttunnel Server. YurttunnelServerAddress = "yurt-tunnel-server-address" - // NetworkingServiceSubnet flag sets the range of IP address for service VIPs. - NetworkingServiceSubnet = "service-cidr" - // NetworkingPodSubnet flag sets the range of IP addresses for the pod network. If set, the control plane will automatically allocate CIDRs for every node. - NetworkingPodSubnet = "pod-network-cidr" + // NetworkingServiceSubnet flag sets the subnet used by kubernetes Services. + NetworkingServiceSubnet = "service-subnet" + // NetworkingPodSubnet flag sets the subnet used by Pods. + NetworkingPodSubnet = "pod-subnet" + // ClusterCIDR flag sets the CIDR range of the pods in the cluster. It is used to bridge traffic coming from outside of the cluster. + ClusterCIDR = "cluster-cidr" + // KubeProxyBindAddress flag sets the IP address for the proxy server to serve on (set to 0.0.0.0 for all interfaces) + KubeProxyBindAddress = "kube-proxy-bind-address" // OpenYurtVersion flag sets the OpenYurt version for the control plane. OpenYurtVersion = "openyurt-version" // K8sVersion flag sets the Kubernetes version for the control plane. K8sVersion = "k8s-version" // ImageRepository flag sets the container registry to pull control plane images from. ImageRepository = "image-repository" - // PassWd flag is the password of master server. + // PassWd flag sets the password of master server. PassWd = "passwd" TmpDownloadDir = "/tmp" @@ -91,6 +95,15 @@ networking: controllerManager: extraArgs: controllers: -nodelifecycle,*,bootstrapsigner,tokencleaner + +--- + +## Custom configurations must specify kind +kind: KubeProxyConfiguration +bindAddress: {{.bind_address}} +clusterCIDR: {{.cluster_cidr}} +featuregates: + EndpointSliceProxying: true ` ) @@ -150,11 +163,11 @@ func addFlags(flagset *flag.FlagSet, o *InitOptions) { "The yurt-tunnel-server address.") flagset.StringVarP( &o.ServiceSubnet, NetworkingServiceSubnet, "", o.ServiceSubnet, - "Use alternative range of IP address for service VIPs.", + "ServiceSubnet is the subnet used by kubernetes Services.", ) flagset.StringVarP( &o.PodSubnet, NetworkingPodSubnet, "", o.PodSubnet, - "Specify range of IP addresses for the pod network. If set, the control plane will automatically allocate CIDRs for every node.", + "PodSubnet is the subnet used by Pods.", ) flagset.StringVarP(&o.Password, PassWd, "p", o.Password, "set master server ssh password", @@ -170,6 +183,12 @@ func addFlags(flagset *flag.FlagSet, o *InitOptions) { flagset.StringVarP(&o.ImageRepository, ImageRepository, "", o.ImageRepository, "Choose a registry to pull cluster images from", ) + flagset.StringVarP(&o.ClusterCIDR, ClusterCIDR, "", o.ClusterCIDR, + "Choose a CIDR range of the pods in the cluster", + ) + flagset.StringVarP(&o.KubeProxyBindAddress, KubeProxyBindAddress, "", o.KubeProxyBindAddress, + "Choose an IP address for the proxy server to serve on", + ) } func NewInitializerWithOptions(o *InitOptions) *clusterInitializer { @@ -257,6 +276,8 @@ func (ci *clusterInitializer) PrepareClusterfile() error { "pod_subnet": ci.PodSubnet, "service_subnet": ci.ServiceSubnet, "yurttunnel_server_address": ci.YurttunnelServerAddress, + "cluster_cidr": ci.ClusterCIDR, + "bind_address": ci.KubeProxyBindAddress, }) if err != nil { return err diff --git a/pkg/yurtadm/cmd/yurtinit/options.go b/pkg/yurtadm/cmd/yurtinit/options.go index 9ed603a08e3..b1cb66464e1 100644 --- a/pkg/yurtadm/cmd/yurtinit/options.go +++ b/pkg/yurtadm/cmd/yurtinit/options.go @@ -32,19 +32,23 @@ type InitOptions struct { YurttunnelServerAddress string ServiceSubnet string PodSubnet string + ClusterCIDR string Password string ImageRepository string OpenYurtVersion string K8sVersion string + KubeProxyBindAddress string } func NewInitOptions() *InitOptions { return &InitOptions{ - ImageRepository: constants.DefaultOpenYurtImageRegistry, - OpenYurtVersion: constants.DefaultOpenYurtVersion, - K8sVersion: constants.DefaultK8sVersion, - PodSubnet: constants.DefaultPodSubnet, - ServiceSubnet: constants.DefaultServiceSubnet, + ImageRepository: constants.DefaultOpenYurtImageRegistry, + OpenYurtVersion: constants.DefaultOpenYurtVersion, + K8sVersion: constants.DefaultK8sVersion, + PodSubnet: constants.DefaultPodSubnet, + ServiceSubnet: constants.DefaultServiceSubnet, + ClusterCIDR: constants.DefaultClusterCIDR, + KubeProxyBindAddress: constants.DefaultKubeProxyBindAddress, } } diff --git a/pkg/yurtadm/constants/constants.go b/pkg/yurtadm/constants/constants.go index 2819fa0bbb1..e20a220cd6f 100644 --- a/pkg/yurtadm/constants/constants.go +++ b/pkg/yurtadm/constants/constants.go @@ -37,6 +37,7 @@ const ( YurthubStaticPodFileName = "yurthub.yaml" PauseImagePath = "registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.2" + OpenYurtCniUrl = "https://github.com/openyurtio/openyurt/releases/download/v0.7.0/openyurt-cni-0.8.7-0.x86_64.rpm" CniUrlFormat = "https://aliacs-edge-k8s-cn-hangzhou.oss-cn-hangzhou.aliyuncs.com/public/pkg/openyurt/cni/%s/cni-plugins-linux-%s-%s.tgz" DefaultKubernetesResourceServer = "dl.k8s.io" KubeUrlFormat = "https://%s/%s/kubernetes-node-linux-%s.tar.gz" @@ -48,18 +49,21 @@ const ( DefaultOpenYurtImageRegistry = "registry.cn-hangzhou.aliyuncs.com/openyurt" DefaultOpenYurtVersion = "latest" - DefaultK8sVersion = "1198" // default kubernetes 1.19.8 + DefaultK8sVersion = "1.19.8" // default kubernetes 1.19.8 DefaultPodSubnet = "10.244.0.0/16" DefaultServiceSubnet = "10.96.0.0/12" - YurtControllerManager = "yurt-controller-manager" - YurtTunnelServer = "yurt-tunnel-server" - YurtTunnelAgent = "yurt-tunnel-agent" - Yurthub = "yurthub" - YurtAppManager = "yurt-app-manager" - YurtAppManagerNamespace = "kube-system" - DirMode = 0755 - FileMode = 0666 - KubeletServiceContent = ` + DefaultClusterCIDR = "10.244.0.0/16" + DefaultKubeProxyBindAddress = "0.0.0.0" + + YurtControllerManager = "yurt-controller-manager" + YurtTunnelServer = "yurt-tunnel-server" + YurtTunnelAgent = "yurt-tunnel-agent" + Yurthub = "yurthub" + YurtAppManager = "yurt-app-manager" + YurtAppManagerNamespace = "kube-system" + DirMode = 0755 + FileMode = 0666 + KubeletServiceContent = ` [Unit] Description=kubelet: The Kubernetes Node Agent Documentation=http://kubernetes.io/docs/ diff --git a/pkg/yurtadm/util/kubernetes/util.go b/pkg/yurtadm/util/kubernetes/util.go index de456060c3c..3d69b5a6051 100644 --- a/pkg/yurtadm/util/kubernetes/util.go +++ b/pkg/yurtadm/util/kubernetes/util.go @@ -127,7 +127,7 @@ func CheckAndInstallKubelet(kubernetesResourceServer, clusterVersion string) err savePath := fmt.Sprintf("%s/kubernetes-node-linux-%s.tar.gz", constants.TmpDownloadDir, runtime.GOARCH) klog.V(1).Infof("Download kubelet from: %s", packageUrl) if err := util.DownloadFile(packageUrl, savePath, 3); err != nil { - return fmt.Errorf("Download kuelet fail: %w", err) + return fmt.Errorf("Download kubelet fail: %w", err) } if err := util.Untar(savePath, constants.TmpDownloadDir); err != nil { return err @@ -145,23 +145,8 @@ func CheckAndInstallKubelet(kubernetesResourceServer, clusterVersion string) err return nil } - ////download and install kubernetes-cni - //cniUrl := fmt.Sprintf(constants.CniUrlFormat, constants.KubeCniVersion, runtime.GOARCH, constants.KubeCniVersion) - //savePath := fmt.Sprintf("%s/cni-plugins-linux-%s-%s.tgz", constants.TmpDownloadDir, runtime.GOARCH, constants.KubeCniVersion) - //klog.V(1).Infof("Download cni from: %s", cniUrl) - //if err := util.DownloadFile(cniUrl, savePath, 3); err != nil { - // return err - //} - // - //if err := os.MkdirAll(constants.KubeCniDir, 0600); err != nil { - // return err - //} - //if err := util.Untar(savePath, constants.KubeCniDir); err != nil { - // return err - //} - - //download and install kubernetes-cni - cniUrl := "https://github.com/openyurtio/openyurt/releases/download/v0.7.0/openyurt-cni-0.8.7-0.x86_64.rpm" + // download and install openyurt-cni + cniUrl := fmt.Sprintf("%s", constants.OpenYurtCniUrl) savePath := fmt.Sprintf("%s/openyurt-cni-0.8.7-0.x86_64.rpm", constants.TmpDownloadDir) klog.V(1).Infof("Download cni from: %s", cniUrl) if err := util.DownloadFile(cniUrl, savePath, 3); err != nil {