From cff39eeb3d0dd3710f83b5eba7bde1afa8fa2d46 Mon Sep 17 00:00:00 2001 From: Ricky Pai Date: Mon, 17 Jul 2017 16:53:18 -0700 Subject: [PATCH 1/2] use init container to install cni on flannel daemonset --- Documentation/kube-flannel.yml | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/Documentation/kube-flannel.yml b/Documentation/kube-flannel.yml index db98fdcde..e76c8d71c 100644 --- a/Documentation/kube-flannel.yml +++ b/Documentation/kube-flannel.yml @@ -53,6 +53,20 @@ spec: operator: Exists effect: NoSchedule serviceAccountName: flannel + initContainers: + - name: install-cni + image: quay.io/coreos/flannel:v0.8.0-amd64 + command: + - cp + args: + - -f + - /etc/kube-flannel/cni-conf.json + - /etc/cni/net.d/10-flannel.conf + volumeMounts: + - name: cni + mountPath: /etc/cni/net.d + - name: flannel-cfg + mountPath: /etc/kube-flannel/ containers: - name: kube-flannel image: quay.io/coreos/flannel:v0.8.0-amd64 @@ -73,14 +87,6 @@ spec: mountPath: /run - name: flannel-cfg mountPath: /etc/kube-flannel/ - - name: install-cni - image: quay.io/coreos/flannel:v0.8.0-amd64 - command: [ "/bin/sh", "-c", "set -e -x; cp -f /etc/kube-flannel/cni-conf.json /etc/cni/net.d/10-flannel.conf; while true; do sleep 3600; done" ] - volumeMounts: - - name: cni - mountPath: /etc/cni/net.d - - name: flannel-cfg - mountPath: /etc/kube-flannel/ volumes: - name: run hostPath: From a154d2f68edd511498c948e33c8cbde20a5901ee Mon Sep 17 00:00:00 2001 From: Tom Denham Date: Fri, 22 Sep 2017 15:06:01 +0100 Subject: [PATCH 2/2] kube-flannel.yml: Update to v0.9.0 and improve docs - Update the manifests to the latest release - Create a new "legacy" manifest for older versions of k8s - Combine the RBAC info into the main manifest --- Documentation/Kubernetes.md | 28 +++--- .../k8s-manifests/kube-flannel-legacy.yml | 89 +++++++++++++++++++ .../{ => k8s-manifests}/kube-flannel-rbac.yml | 2 +- Documentation/kube-flannel-aliyun.yml | 77 +++++++++++++--- Documentation/kube-flannel.yml | 42 ++++++++- Documentation/troubleshooting.md | 4 +- README.md | 3 + 7 files changed, 214 insertions(+), 31 deletions(-) create mode 100644 Documentation/k8s-manifests/kube-flannel-legacy.yml rename Documentation/{ => k8s-manifests}/kube-flannel-rbac.yml (91%) diff --git a/Documentation/Kubernetes.md b/Documentation/Kubernetes.md index c7a89eee7..fd2cd84e2 100644 --- a/Documentation/Kubernetes.md +++ b/Documentation/Kubernetes.md @@ -4,24 +4,28 @@ For information on deploying flannel manually, using the Kubernetes installer to NOTE: If `kubeadm` is used, then pass `--pod-network-cidr=10.244.0.0/16` to `kubeadm init` to ensure that the `podCIDR` is set. -kubeadm has RBAC enabled by default so you must apply the `kube-flannel-rbac.yml` manifest as well as the `kube-flannel.yml` manifest. +# kube-flannel.yaml -* `kubectl apply -f kube-flannel-rbac.yml -f kube-flannel.yml` +The `flannel` manifest defines four things: +1. A ClusterRole and ClusterRoleBinading for role based acccess control (RBAC). +2. A service account for `flannel` to use. +3. A ConfigMap containing both a CNI configuration and a `flannel` configuration. The `network` in the `flannel` configuration should match the pod network CIDR. The choice of `backend` is also made here and defaults to VXLAN. +4. A DaemonSet to deploy the `flannel` pod on each Node. The pod has two containers 1) the `flannel` daemon itself, and 2) an initContainer for deploying the CNI configuration to a location that the `kubelet` can read. -If you didn't apply the `kube-flannel-rbac.yml` manifest, you'll see errors in your flanneld logs about failing to connect. -* `Failed to create SubnetManager: error retrieving pod spec...` +When you run pods, they will be allocated IP addresses from the pod network CIDR. No matter which node those pods end up on, they will be able to communicate with each other. -If you forgot to apply the `kube-flannel-rbac.yml` manifest and notice that flannel fails to start, then it is safe to just apply the `kube-flannel-rbac.yml` manifest without running `kubectl delete -f kube-flannel.yaml` first. -* `kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel-rbac.yml` +## Older versions of Kubernetes -# kube-flannel.yaml +`kube-flannel.yaml` has some features that aren't compatible with older versions of Kubernetes, though flanneld itself should work with any version of Kubernetes. -The `flannel` manifest defines three things: -1. A service account for `flannel` to use. -2. A ConfigMap containing both a CNI configuration and a `flannel` configuration. The `network` in the `flannel` configuration should match the pod network CIDR. The choice of `backend` is also made here and defaults to VXLAN. -3. A DaemonSet to deploy the `flannel` pod on each Node. The pod has two containers 1) the `flannel` daemon itself, and 2) a container for deploying the CNI configuration to a location that the `kubelet` can read. +If you see errors saying `found invalid field...` when you try to apply `kube-flannel.yaml` then you can try the "legacy" manifest file +* `kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/k8s-manifests/kube-flannel-legacy.yml` -When you run pods, they will be allocated IP addresses from the pod network CIDR. No matter which node those pods end up on, they will be able to communicate with each other. +This file does not bundle RBAC permissions. If you need those, run +* `kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/k8s-manifests/kube-flannel-rbac.yml` + +If you didn't apply the `kube-flannel-rbac.yml` manifest and you need to, you'll see errors in your flanneld logs about failing to connect. +* `Failed to create SubnetManager: error retrieving pod spec...` ## The flannel CNI plugin diff --git a/Documentation/k8s-manifests/kube-flannel-legacy.yml b/Documentation/k8s-manifests/kube-flannel-legacy.yml new file mode 100644 index 000000000..d8c6eb994 --- /dev/null +++ b/Documentation/k8s-manifests/kube-flannel-legacy.yml @@ -0,0 +1,89 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flannel + namespace: kube-system +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: kube-flannel-cfg + namespace: kube-system + labels: + tier: node + app: flannel +data: + cni-conf.json: | + { + "name": "cbr0", + "type": "flannel", + "delegate": { + "isDefaultGateway": true + } + } + net-conf.json: | + { + "Network": "10.244.0.0/16", + "Backend": { + "Type": "vxlan" + } + } +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: kube-flannel-ds + namespace: kube-system + labels: + tier: node + app: flannel +spec: + template: + metadata: + labels: + tier: node + app: flannel + spec: + hostNetwork: true + nodeSelector: + beta.kubernetes.io/arch: amd64 + serviceAccountName: flannel + containers: + - name: kube-flannel + image: quay.io/coreos/flannel:v0.9.0-amd64 + command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr" ] + securityContext: + privileged: true + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: run + mountPath: /run + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + - name: install-cni + image: quay.io/coreos/flannel:v0.9.0-amd64 + command: [ "/bin/sh", "-c", "set -e -x; cp -f /etc/kube-flannel/cni-conf.json /etc/cni/net.d/10-flannel.conf; while true; do sleep 3600; done" ] + volumeMounts: + - name: cni + mountPath: /etc/cni/net.d + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + volumes: + - name: run + hostPath: + path: /run + - name: cni + hostPath: + path: /etc/cni/net.d + - name: flannel-cfg + configMap: + name: kube-flannel-cfg \ No newline at end of file diff --git a/Documentation/kube-flannel-rbac.yml b/Documentation/k8s-manifests/kube-flannel-rbac.yml similarity index 91% rename from Documentation/kube-flannel-rbac.yml rename to Documentation/k8s-manifests/kube-flannel-rbac.yml index d66465cdd..d5052b679 100644 --- a/Documentation/kube-flannel-rbac.yml +++ b/Documentation/k8s-manifests/kube-flannel-rbac.yml @@ -1,7 +1,7 @@ # Create the clusterrole and clusterrolebinding: # $ kubectl create -f kube-flannel-rbac.yml # Create the pod using the same namespace used by the flannel serviceaccount: -# $ kubectl create --namespace kube-system -f kube-flannel.yml +# $ kubectl create --namespace kube-system -f kube-flannel-legacy.yml --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1beta1 diff --git a/Documentation/kube-flannel-aliyun.yml b/Documentation/kube-flannel-aliyun.yml index 49145eded..f2d8fff04 100644 --- a/Documentation/kube-flannel-aliyun.yml +++ b/Documentation/kube-flannel-aliyun.yml @@ -1,4 +1,48 @@ --- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: flannel +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: flannel +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flannel +subjects: +- kind: ServiceAccount + name: flannel + namespace: kube-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flannel + namespace: kube-system +--- kind: ConfigMap apiVersion: v1 metadata: @@ -42,9 +86,28 @@ spec: hostNetwork: true nodeSelector: beta.kubernetes.io/arch: amd64 + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + serviceAccountName: flannel + initContainers: + - name: install-cni + image: registry.cn-hangzhou.aliyuncs.com/google-containers/flannel:0.9.0 + command: + - cp + args: + - -f + - /etc/kube-flannel/cni-conf.json + - /etc/cni/net.d/10-flannel.conf + volumeMounts: + - name: cni + mountPath: /etc/cni/net.d + - name: flannel-cfg + mountPath: /etc/kube-flannel/ containers: - name: kube-flannel - image: registry.cn-hangzhou.aliyuncs.com/google-containers/flannel:0.7.0 + image: registry.cn-hangzhou.aliyuncs.com/google-containers/flannel:0.9.0 command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr" ] securityContext: privileged: true @@ -57,23 +120,11 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - - name: ACCESS_KEY_ID - value: [replace with your own key] - - name: ACCESS_KEY_SECRET - value: [replace with your own secret] volumeMounts: - name: run mountPath: /run - name: flannel-cfg mountPath: /etc/kube-flannel/ - - name: install-cni - image: registry.cn-hangzhou.aliyuncs.com/google-containers/flannel:0.7.0 - command: [ "/bin/sh", "-c", "set -e -x; cp -f /etc/kube-flannel/cni-conf.json /etc/cni/net.d/10-flannel.conf; while true; do sleep 3600; done" ] - volumeMounts: - - name: cni - mountPath: /etc/cni/net.d - - name: flannel-cfg - mountPath: /etc/kube-flannel/ volumes: - name: run hostPath: diff --git a/Documentation/kube-flannel.yml b/Documentation/kube-flannel.yml index e76c8d71c..d3714cec8 100644 --- a/Documentation/kube-flannel.yml +++ b/Documentation/kube-flannel.yml @@ -1,4 +1,42 @@ --- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: flannel +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: flannel +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flannel +subjects: +- kind: ServiceAccount + name: flannel + namespace: kube-system +--- apiVersion: v1 kind: ServiceAccount metadata: @@ -55,7 +93,7 @@ spec: serviceAccountName: flannel initContainers: - name: install-cni - image: quay.io/coreos/flannel:v0.8.0-amd64 + image: quay.io/coreos/flannel:v0.9.0-amd64 command: - cp args: @@ -69,7 +107,7 @@ spec: mountPath: /etc/kube-flannel/ containers: - name: kube-flannel - image: quay.io/coreos/flannel:v0.8.0-amd64 + image: quay.io/coreos/flannel:v0.9.0-amd64 command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr" ] securityContext: privileged: true diff --git a/Documentation/troubleshooting.md b/Documentation/troubleshooting.md index 2f16d2e9b..046bfacd9 100644 --- a/Documentation/troubleshooting.md +++ b/Documentation/troubleshooting.md @@ -79,6 +79,4 @@ It's possible to manually set the `podCIDR` for each node. * `failed to read net conf` - flannel expects to be able to read the net conf from "/etc/kube-flannel/net-conf.json". In the provided manifest, this is set up in the `kube-flannel-cfg` ConfigMap. * `error parsing subnet config` - The net conf is malformed. Double check that it has the right content and is valid JSON. * `node pod cidr not assigned` - The node doesn't have a `podCIDR` defined. See above for more info. -* `Failed to create SubnetManager: error retrieving pod spec for 'kube-system/kube-flannel-ds-abc123': the server does not allow access to the requested resource` - The kubernetes cluster has RBAC enabled. Run `https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel-rbac.yml` - - +* `Failed to create SubnetManager: error retrieving pod spec for 'kube-system/kube-flannel-ds-abc123': the server does not allow access to the requested resource` - The kubernetes cluster has RBAC enabled. Run `https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel-rbac.yml` \ No newline at end of file diff --git a/README.md b/README.md index 82396e8dd..c237a223f 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,9 @@ Though not required, it's recommended that flannel uses the Kubernetes API as it Flannel can be added to any existing Kubernetes cluster though it's simplest to add `flannel` before any pods using the pod network have been started. +For Kubernetes v1.6+ +`kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml` + See [Kubernetes](Documentation/Kubernetes.md) for more details. ## Getting started on Docker