adding ovn4nfv-k8s-plugin in kubespray

.gitlab-ci/packet.yml

@@ -110,6 +110,11 @@ packet_opensuse-canal:
   extends: .packet
   when: on_success
 
+packet_ubuntu18-ovn4nfv:
+  stage: deploy-part2
+  extends: .packet
+  when: on_success
+
 # Contiv does not work in k8s v1.16
 # packet_ubuntu16-contiv-sep:
 #   stage: deploy-part2

README.md

@@ -130,6 +130,7 @@ Note: Upstart/SysV init based OS types are not supported.
   - [kube-ovn](https://github.com/alauda/kube-ovn) v1.2.1
   - [kube-router](https://github.com/cloudnativelabs/kube-router) v1.0.0
   - [multus](https://github.com/intel/multus-cni) v3.6.0
+  - [ovn4nfv](https://github.com/opnfv/ovn4nfv-k8s-plugin) v1.0.0
   - [weave](https://github.com/weaveworks/weave) v2.6.5
 - Application
   - [ambassador](https://github.com/datawire/ambassador): v1.5
@@ -180,6 +181,8 @@ You can choose between 10 network plugins. (default: `calico`, except Vagrant us
 - [contiv](docs/contiv.md): supports vlan, vxlan, bgp and Cisco SDN networking. This plugin is able to
     apply firewall policies, segregate containers in multiple network and bridging pods onto physical networks.
 
+- [ovn4nfv](docs/ovn4nfv.md): [ovn4nfv-k8s-plugins](https://github.com/opnfv/ovn4nfv-k8s-plugin) is the network controller, OVS agent and CNI server to offer basic SFC and OVN overlay networking.
+
 - [weave](docs/weave.md): Weave is a lightweight container overlay network that doesn't require an external K/V database cluster.
     (Please refer to `weave` [troubleshooting documentation](https://www.weave.works/docs/net/latest/troubleshooting/)).
 

docs/ci.md

@@ -4,51 +4,51 @@ To generate this Matrix run `./tests/scripts/md-table/main.py`
 
 ## docker
 
-| OS / CNI | calico | canal | cilium | contiv | flannel | kube-ovn | kube-router | macvlan | weave |
-|---| --- | --- | --- | --- | --- | --- | --- | --- | --- |
-amazon |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-centos7 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :white_check_mark: |
-centos8 |  :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: |
-debian10 |  :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-debian9 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: |
-fedora31 |  :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: |
-fedora32 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: |
-opensuse |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-oracle7 |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu16 |  :x: | :white_check_mark: | :x: | :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: |
-ubuntu18 |  :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: |
-ubuntu20 |  :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: |
+| OS / CNI | calico | canal | cilium | contiv | flannel | kube-ovn | kube-router | macvlan | ovn4nfv | weave |
+|---| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+amazon |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+centos7 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :white_check_mark: |
+centos8 |  :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: |
+debian10 |  :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian9 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: |
+fedora31 |  :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
+fedora32 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: |
+opensuse |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+oracle7 |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu16 |  :x: | :white_check_mark: | :x: | :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :white_check_mark: |
+ubuntu18 |  :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :white_check_mark: |
+ubuntu20 |  :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
 
 ## crio
 
-| OS / CNI | calico | canal | cilium | contiv | flannel | kube-ovn | kube-router | macvlan | weave |
-|---| --- | --- | --- | --- | --- | --- | --- | --- | --- |
-amazon |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-centos7 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-centos8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-debian10 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-debian9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-fedora31 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-fedora32 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-opensuse |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-oracle7 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu16 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu18 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu20 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+| OS / CNI | calico | canal | cilium | contiv | flannel | kube-ovn | kube-router | macvlan | ovn4nfv | weave |
+|---| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+amazon |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+centos7 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+centos8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian10 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+fedora31 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+fedora32 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+opensuse |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+oracle7 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu16 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu18 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu20 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 
 ## containerd
 
-| OS / CNI | calico | canal | cilium | contiv | flannel | kube-ovn | kube-router | macvlan | weave |
-|---| --- | --- | --- | --- | --- | --- | --- | --- | --- |
-amazon |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-centos7 |  :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: |
-centos8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-debian10 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-debian9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-fedora31 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-fedora32 |  :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: |
-opensuse |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-oracle7 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu16 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu18 |  :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: |
-ubuntu20 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+| OS / CNI | calico | canal | cilium | contiv | flannel | kube-ovn | kube-router | macvlan | ovn4nfv | weave |
+|---| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+amazon |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+centos7 |  :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
+centos8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian10 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+fedora31 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+fedora32 |  :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: |
+opensuse |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+oracle7 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu16 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu18 |  :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
+ubuntu20 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |

docs/ovn4nfv.md

@@ -0,0 +1,49 @@
+# OVN4NFV-k8S-Plugin
+
+Intro to [ovn4nfv-k8s-plugin](https://github.com/opnfv/ovn4nfv-k8s-plugin)
+
+## How to use it
+
+* Enable ovn4nfv in `group_vars/k8s-cluster/k8s-cluster.yml`
+
+```yml
+...
+kube_network_plugin: ovn4nfv
+...
+```
+
+## Verifying ovn4nfv kube network plugin
+
+* ovn4nfv install ovn control plan in the master and ovn daemonset in all nodes
+* Network function Networking(nfn) operator is install in the master and nfn agent is installed in all the node
+* ovn4nfv install `ovn4nfvk8s-cni` cni shim binary in `/opt/cni/bin/` and nfn agent act as the cni server
+* All ovn4nfv pods are installed in the kube-system
+
+```ShellSession
+# From K8s client
+# kubectl get pods -n kube-system -l app=ovn-control-plane -o wide
+NAME                                 READY   STATUS    RESTARTS   AGE     IP               NODE     NOMINATED NODE   READINESS GATES
+ovn-control-plane-5f8b7bcc65-w759g   1/1     Running   0          3d18h   192.168.121.25   master   <none>           <none>
+
+# kubectl get pods -n kube-system -l app=ovn-controller -o wide
+NAME                   READY   STATUS    RESTARTS   AGE     IP               NODE       NOMINATED NODE   READINESS GATES
+ovn-controller-54zzj   1/1     Running   0          3d18h   192.168.121.24   minion01   <none>           <none>
+ovn-controller-7cljt   1/1     Running   0          3d18h   192.168.121.25   master     <none>           <none>
+ovn-controller-cx46g   1/1     Running   0          3d18h   192.168.121.15   minion02   <none>           <none>
+
+# kubectl get pods -n kube-system -l name=nfn-operator -o wide
+NAME                            READY   STATUS    RESTARTS   AGE     IP               NODE     NOMINATED NODE   READINESS GATES
+nfn-operator-6dc44dbf48-xk9zl   1/1     Running   0          3d18h   192.168.121.25   master   <none>           <none>
+
+# kubectl get pods -n kube-system -l app=nfn-agent -o wide
+NAME              READY   STATUS    RESTARTS   AGE     IP               NODE       NOMINATED NODE   READINESS GATES
+nfn-agent-dzlpp   1/1     Running   0          3d18h   192.168.121.15   minion02   <none>           <none>
+nfn-agent-jcdbn   1/1     Running   0          3d18h   192.168.121.25   master     <none>           <none>
+nfn-agent-lrkzk   1/1     Running   0          3d18h   192.168.121.24   minion01   <none>           <none>
+
+# kubectl get pods -n kube-system -l app=ovn4nfv -o wide
+NAME                READY   STATUS    RESTARTS   AGE     IP               NODE       NOMINATED NODE   READINESS GATES
+ovn4nfv-cni-5zdz2   1/1     Running   0          3d18h   192.168.121.24   minion01   <none>           <none>
+ovn4nfv-cni-k5wjp   1/1     Running   0          3d18h   192.168.121.25   master     <none>           <none>
+ovn4nfv-cni-t6z5b   1/1     Running   0          3d18h   192.168.121.15   minion02   <none>           <none>
+```

roles/download/defaults/main.yml

@@ -83,6 +83,8 @@ cilium_version: "v1.8.1"
 kube_ovn_version: "v1.2.1"
 kube_router_version: "v1.0.0"
 multus_version: "v3.6"
+ovn4nfv_ovn_image_version: "v1.0.0"
+ovn4nfv_k8s_plugin_image_version: "v1.0.0"
 
 # Get kubernetes major version (i.e. 1.17.4 => 1.17)
 kube_major_version: "{{ kube_version | regex_replace('^v([0-9])+\\.([0-9]+)\\.[0-9]+', 'v\\1.\\2') }}"
@@ -490,6 +492,10 @@ kube_router_image_repo: "{{ docker_image_repo }}/cloudnativelabs/kube-router"
 kube_router_image_tag: "{{ kube_router_version }}"
 multus_image_repo: "{{ docker_image_repo }}/nfvpe/multus"
 multus_image_tag: "{{ multus_version }}"
+ovn4nfv_ovn_image_repo: "{{ docker_image_repo }}/integratedcloudnative/ovn-images"
+ovn4nfv_ovn_image_tag: "{{ ovn4nfv_ovn_image_version }}"
+ovn4nfv_k8s_plugin_image_repo: "{{ docker_image_repo }}/integratedcloudnative/ovn4nfv-k8s-plugin"
+ovn4nfv_k8s_plugin_image_tag: "{{ ovn4nfv_k8s_plugin_image_version }}"
 
 nginx_image_repo: "{{ docker_image_repo }}/library/nginx"
 nginx_image_tag: 1.19
@@ -812,6 +818,15 @@ downloads:
     groups:
     - k8s-cluster
 
+  ovn4nfv:
+    enabled: "{{ kube_network_plugin == 'ovn4nfv' }}"
+    container: true
+    repo: "{{ ovn4nfv_k8s_plugin_image_repo }}"
+    tag: "{{ ovn4nfv_k8s_plugin_image_tag }}"
+    sha256: "{{ ovn4nfv_k8s_plugin_digest_checksum|default(None) }}"
+    groups:
+    - k8s-cluster
+
   contiv:
     enabled: "{{ kube_network_plugin == 'contiv' }}"
     container: true

roles/kubernetes-apps/network_plugin/meta/main.yml

@@ -40,6 +40,11 @@ dependencies:
     tags:
       - kube-router
 
+  - role: kubernetes-apps/network_plugin/ovn4nfv
+    when: kube_network_plugin == 'ovn4nfv'
+    tags:
+      - ovn4nfv
+
   - role: kubernetes-apps/network_plugin/multus
     when: kube_network_plugin_multus
     tags:

roles/kubernetes-apps/network_plugin/ovn4nfv/tasks/main.yml

@@ -0,0 +1,9 @@
+---
+- name: ovn4nfv-k8s | Start Resources
+  kube:
+    name: "{{ item.item.name }}"
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/{{ item.item.file }}"
+    state: "latest"
+  with_items: "{{ ovn4nfv_node_manifests.results }}"
+  when: inventory_hostname == groups['kube-master'][0] and not item is skipped

roles/kubernetes/node/templates/kubelet.env.v1beta1.j2

@@ -37,7 +37,7 @@ KUBELET_ARGS="{{ kubelet_args_base }} {% if node_taints|default([]) %}--register
 {% if kubelet_flexvolumes_plugins_dir is defined %}
 KUBELET_VOLUME_PLUGIN="--volume-plugin-dir={{ kubelet_flexvolumes_plugins_dir }}"
 {% endif %}
-{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "cni", "flannel", "weave", "contiv", "cilium", "kube-ovn", "kube-router", "macvlan"] %}
+{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "cni", "flannel", "weave", "contiv", "cilium", "kube-ovn", "ovn4nfv", "kube-router", "macvlan"] %}
 KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
 {% elif kube_network_plugin is defined and kube_network_plugin == "cloud" %}
 KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kubenet"

roles/kubernetes/preinstall/tasks/0020-verify-settings.yml

@@ -22,7 +22,7 @@
 
 - name: Stop if unknown network plugin
   assert:
-    that: kube_network_plugin in ['calico', 'canal', 'flannel', 'weave', 'cloud', 'cilium', 'cni', 'contiv', 'kube-ovn', 'kube-router', 'macvlan']
+    that: kube_network_plugin in ['calico', 'canal', 'flannel', 'weave', 'cloud', 'cilium', 'cni', 'contiv', 'ovn4nfv','kube-ovn', 'kube-router', 'macvlan']
     msg: "{{ kube_network_plugin }} is not supported"
   when:
     - kube_network_plugin is defined

roles/kubernetes/preinstall/tasks/0050-create_directories.yml

@@ -52,7 +52,7 @@
     - "/opt/cni/bin"
     - "/var/lib/calico"
   when:
-    - kube_network_plugin in ["calico", "weave", "canal", "flannel", "contiv", "cilium", "kube-ovn", "kube-router", "macvlan"]
+    - kube_network_plugin in ["calico", "weave", "canal", "flannel", "contiv", "cilium", "kube-ovn", "ovn4nfv", "kube-router", "macvlan"]
     - inventory_hostname in groups['k8s-cluster']
   tags:
     - network
@@ -61,6 +61,7 @@
     - weave
     - canal
     - contiv
+    - ovn4nfv
     - kube-ovn
     - kube-router
     - bootstrap-os

roles/network_plugin/meta/main.yml

@@ -50,6 +50,11 @@ dependencies:
     tags:
       - kube-router
 
+  - role: network_plugin/ovn4nfv
+    when: kube_network_plugin == 'ovn4nfv'
+    tags:
+      - ovn4nfv
+
   - role: network_plugin/multus
     when: kube_network_plugin_multus
     tags:

roles/network_plugin/ovn4nfv/defaults/main.yml

@@ -0,0 +1,15 @@
+---
+ovn_control_plane_cpu_request: 500m
+ovn_control_plane_memory_request: 300Mi
+ovn_controller_cpu_request: 200m
+ovn_controller_memory_request: 300Mi
+ovn_controller_cpu_limit: 1000m
+ovn_controller_memory_limit: 800Mi
+ovn4nfv_cni_cpu_request: 100m
+ovn4nfv_cni_memory_request: 50Mi
+ovn4nfv_cni_cpu_limit: 100m
+ovn4nfv_cni_memory_limit: 50Mi
+nfn_agent_cpu_request: 100m
+nfn_agent_memory_request: 50Mi
+nfn_agent_cpu_limit: 100m
+nfn_agent_memory_limit: 50Mi

roles/network_plugin/ovn4nfv/tasks/main.yml

@@ -0,0 +1,15 @@
+---
+- name: ovn4nfv | Label control-plane node
+  command: >-
+    {{ bin_dir }}/kubectl label --overwrite node {{ groups['kube-master'] | first }} ovn4nfv-k8s-plugin=ovn-control-plane
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+
+- name: ovn4nfv | Create ovn4nfv-k8s manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+  with_items:
+    - {name: ovn-daemonset, file: ovn-daemonset.yml}
+    - {name: ovn4nfv-k8s-plugin, file: ovn4nfv-k8s-plugin.yml}
+  register: ovn4nfv_node_manifests