Support extra_network and extra_subnet labels

This allows users to add extra network interface to cluster nodes, e.g. storage network. Story: 2002448 Task: 21983 Co-Authored-By: Bharat Kunwar <bharat@stackhpc.com> Change-Id: I10a6a4d72e9ec635f2c73d9fe64a8d136228f532
stackhpc · Jan 31, 2022 · bea8c72 · bea8c72
1 parent f98cdce
commit bea8c72
Show file tree

Hide file tree

Showing 7 changed files with 195 additions and 7 deletions.
diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst
@@ -502,6 +502,12 @@ the table are linked to more details elsewhere in the user guide.
 +---------------------------------------+--------------------+---------------+
 | `fixed_subnet_cidr`_                  | see below          | ""            |
 +---------------------------------------+--------------------+---------------+
+| `extra_network`_                      | see below          | ""            |
++---------------------------------------+--------------------+---------------+
+| `extra_subnet`_                       | see below          | ""            |
++---------------------------------------+--------------------+---------------+
+| `extra_security_group`_               | see below          | see below     |
++---------------------------------------+--------------------+---------------+
 
 .. _cluster:
 
@@ -1635,6 +1641,22 @@ _`fixed_subnet_cidr`
   specified an existing fixed_subnet during cluster creation.
   Ussuri default: 10.0.0.0/24
 
+_`extra_network`
+  Optional additional network name or UUID to add to cluster nodes.
+  When not specified, additional networks are not added. Optionally specify
+  'extra_subnet' if you wish to use a specific subnet on the network.
+  Default: ""
+
+_`extra_subnet`
+  Optional additional subnet name or UUID to add to cluster nodes.
+  Only used when 'extra_network' is defined.
+  Default: ""
+
+_`extra_security_group`
+  Optional additional group name or UUID to add to network port.
+  Only used when 'extra_network' is defined. 
+  Default: cluster node default security group.
+
 External load balancer for services
 -----------------------------------
 
@@ -2723,7 +2745,6 @@ _`calico_tag`
   Victoria default: v3.13.1
   Wallaby default: v3.18.0
 
-
 Besides, the Calico network driver needs kube_tag with v1.9.3 or later, because
 Calico needs extra mounts for the kubelet container. See `commit
 <https://github.com/projectatomic/atomic-system-containers/commit/54ab8abc7fa1bfb6fa674f55cd0c2fa0c812fd36>`_

diff --git a/magnum/drivers/heat/k8s_fedora_template_def.py b/magnum/drivers/heat/k8s_fedora_template_def.py
@@ -126,7 +126,8 @@ def get_params(self, context, cluster_template, cluster, **kwargs):
                       'min_node_count', 'max_node_count', 'npd_enabled',
                       'ostree_remote', 'ostree_commit',
                       'use_podman', 'kube_image_digest',
-                      'metrics_scraper_tag']
+                      'metrics_scraper_tag',
+                      'extra_network', 'extra_subnet', 'extra_security_group']
 
         labels = self._get_relevant_labels(cluster, kwargs)
 

diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml
@@ -984,13 +984,32 @@ parameters:
     description: >
       Post install manifest URL used to setup some cloud provider/vendor
       specific configs
-    default: ""
+    default: ''
 
   master_lb_allowed_cidrs:
     type: comma_delimited_list
     description: The allowed CIDR list for master load balancer
     default: []
 
+  extra_network:
+    type: string
+    description: >
+      Additional network to bind nodes to
+    default: ''
+
+  extra_subnet:
+    type: string
+    description: >
+      Subnet for additional network
+    default: ''
+
+  extra_security_group:
+    type: string
+    description: >
+      Additional security group name
+    default: ''
+
+
 resources:
 
   ######################################################################
@@ -1373,6 +1392,9 @@ resources:
           containerd_tarball_sha256: {get_param: containerd_tarball_sha256}
           post_install_manifest_url: {get_param: post_install_manifest_url}
           metrics_scraper_tag: {get_param: metrics_scraper_tag}
+          extra_network: {get_param: extra_network}
+          extra_subnet: {get_param: extra_subnet}
+          extra_security_group: {get_param: extra_security_group}
 
   kube_cluster_config:
     condition: create_cluster_resources
@@ -1552,6 +1574,10 @@ resources:
           containerd_tarball_sha256: {get_param: containerd_tarball_sha256}
           kube_service_account_key: {get_param: kube_service_account_key}
           kube_service_account_private_key: {get_param: kube_service_account_private_key}
+          extra_network: {get_param: extra_network}
+          extra_subnet: {get_param: extra_subnet}
+          extra_security_group: {get_param: extra_security_group}
+
 outputs:
 
   api_address:

diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml
@@ -692,6 +692,21 @@ parameters:
     description: >
       Tag of metrics-scraper for kubernetes dashboard.
 
+  extra_network:
+    type: string
+    description: >
+      Additional network name to bind ports to instances
+
+  extra_subnet:
+    type: string
+    description: >
+      Additional subnet name
+
+  extra_security_group:
+    type: string
+    description: >
+      Additional security group name
+
 conditions:
 
   image_based: {equals: [{get_param: boot_volume_size}, 0]}
@@ -701,6 +716,24 @@ conditions:
       - get_param: boot_volume_size
       - 0
 
+  extra_network_defined:
+    not:
+      equals:
+      - get_param: extra_network
+      - ''
+
+  extra_subnet_defined:
+    not:
+      equals:
+      - get_param: extra_subnet
+      - ''
+
+  extra_security_group_defined:
+    not:
+      equals:
+      - get_param: extra_security_group
+      - ''
+
 resources:
   ######################################################################
   #
@@ -959,7 +992,12 @@ resources:
       software_config_transport: POLL_SERVER_HEAT
       user_data: {get_resource: agent_config}
       networks:
-        - port: {get_resource: kube_master_eth0}
+        list_concat:
+          - - port: {get_resource: kube_master_eth0}
+          - if:
+              - "extra_network_defined"
+              - - port: {get_resource: kube_master_eth1}
+              - []
       scheduler_hints: { group: { get_param: nodes_server_group_id }}
       availability_zone: {get_param: availability_zone}
 
@@ -973,7 +1011,12 @@ resources:
       software_config_transport: POLL_SERVER_HEAT
       user_data: {get_resource: agent_config}
       networks:
-        - port: {get_resource: kube_master_eth0}
+        list_concat:
+          - - port: {get_resource: kube_master_eth0}
+          - if:
+              - "extra_network_defined"
+              - - port: {get_resource: kube_master_eth1}
+              - []
       scheduler_hints: { group: { get_param: nodes_server_group_id }}
       availability_zone: {get_param: availability_zone}
       block_device_mapping_v2:
@@ -992,6 +1035,23 @@ resources:
         - ip_address: {get_param: pods_network_cidr}
       replacement_policy: AUTO
 
+  kube_master_eth1:
+    type: OS::Neutron::Port
+    condition: extra_network_defined
+    properties:
+      network: {get_param: extra_network}
+      security_groups:
+        - if:
+            - "extra_security_group_defined"
+            - {get_param: extra_security_group}
+            - {get_param: secgroup_kube_master_id}
+      fixed_ips:
+        if:
+          - "extra_subnet_defined"
+          - - subnet: {get_param: extra_subnet}
+          - []
+      replacement_policy: AUTO
+
   kube_master_floating:
     type: Magnum::Optional::KubeMaster::Neutron::FloatingIP
     properties:

diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubeminion.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubeminion.yaml
@@ -369,6 +369,21 @@ parameters:
       The private key will be used to sign generated k8s service account
       tokens.
 
+  extra_network:
+    type: string
+    description: >
+      Additional network name to bind ports to instances
+
+  extra_subnet:
+    type: string
+    description: >
+      Additional subnet name
+
+  extra_security_group:
+    type: string
+    description: >
+      Additional seurity group name
+
 conditions:
 
   image_based: {equals: [{get_param: boot_volume_size}, 0]}
@@ -378,6 +393,25 @@ conditions:
       - get_param: boot_volume_size
       - 0
 
+  extra_network_defined:
+    not:
+      equals:
+      - get_param: extra_network
+      - ''
+
+  extra_subnet_defined:
+    not:
+      equals:
+      - get_param: extra_subnet
+      - ''
+
+  extra_security_group_defined:
+    not:
+      equals:
+      - get_param: extra_security_group
+      - ''
+
+
 resources:
 
   agent_config:
@@ -542,7 +576,12 @@ resources:
       user_data_format: SOFTWARE_CONFIG
       software_config_transport: POLL_SERVER_HEAT
       networks:
-        - port: {get_resource: kube_minion_eth0}
+        list_concat:
+          - - port: {get_resource: kube_minion_eth0}
+          - if:
+              - "extra_network_defined"
+              - - port: {get_resource: kube_minion_eth1}
+              - []
       scheduler_hints: { group: { get_param: nodes_server_group_id }}
       availability_zone: {get_param: availability_zone}
 
@@ -556,7 +595,12 @@ resources:
       user_data_format: SOFTWARE_CONFIG
       software_config_transport: POLL_SERVER_HEAT
       networks:
-        - port: {get_resource: kube_minion_eth0}
+        list_concat:
+          - - port: {get_resource: kube_minion_eth0}
+          - if:
+              - "extra_network_defined"
+              - - port: {get_resource: kube_minion_eth1}
+              - []
       scheduler_hints: { group: { get_param: nodes_server_group_id }}
       availability_zone: {get_param: availability_zone}
       block_device_mapping_v2:
@@ -575,6 +619,23 @@ resources:
         - ip_address: {get_param: pods_network_cidr}
       replacement_policy: AUTO
 
+  kube_minion_eth1:
+    type: OS::Neutron::Port
+    condition: extra_network_defined
+    properties:
+      network: {get_param: extra_network}
+      security_groups:
+        - if:
+            - "extra_security_group_defined"
+            - get_param: extra_security_group
+            - get_param: secgroup_kube_minion_id
+      fixed_ips:
+        if:
+          - "extra_subnet_defined"
+          - - subnet: {get_param: extra_subnet}
+          - []
+      replacement_policy: AUTO
+
   kube_minion_floating:
     type: Magnum::Optional::KubeMinion::Neutron::FloatingIP
     properties:

diff --git a/magnum/tests/unit/drivers/test_template_definition.py b/magnum/tests/unit/drivers/test_template_definition.py
@@ -622,6 +622,9 @@ def test_k8s_get_params(self, mock_generate_csr_and_key,
         metrics_scraper_tag = mock_cluster.labels.get('metrics_scraper_tag')
         master_lb_allowed_cidrs = mock_cluster.labels.get(
             'master_lb_allowed_cidrs')
+        extra_network = mock_cluster.labels.get('extra_network')
+        extra_subnet = mock_cluster.labels.get('extra_subnet')
+        extra_security_group = mock_cluster.labels.get('extra_security_group')
 
         k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
 
@@ -746,6 +749,9 @@ def test_k8s_get_params(self, mock_generate_csr_and_key,
             'metrics_scraper_tag': metrics_scraper_tag,
             'master_lb_allowed_cidrs': master_lb_allowed_cidrs,
             'fixed_subnet_cidr': '20.200.0.0/16',
+            'extra_network': extra_network,
+            'extra_subnet': extra_subnet,
+            'extra_security_group': extra_security_group,
         }}
         mock_get_params.assert_called_once_with(mock_context,
                                                 mock_cluster_template,
@@ -1177,6 +1183,10 @@ def test_k8s_get_params_insecure(self, mock_generate_csr_and_key,
 
         master_lb_allowed_cidrs = mock_cluster.labels.get(
             'master_lb_allowed_cidrs')
+        extra_network = mock_cluster.labels.get('extra_network')
+        extra_subnet = mock_cluster.labels.get('extra_subnet')
+        extra_security_group = mock_cluster.labels.get('extra_security_group')
+
 
         k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
 
@@ -1303,6 +1313,9 @@ def test_k8s_get_params_insecure(self, mock_generate_csr_and_key,
             'metrics_scraper_tag': metrics_scraper_tag,
             'master_lb_allowed_cidrs': master_lb_allowed_cidrs,
             'fixed_subnet_cidr': '20.200.0.0/16',
+            'extra_network': extra_network,
+            'extra_subnet': extra_subnet,
+            'extra_security_group': extra_security_group,
         }}
         mock_get_params.assert_called_once_with(mock_context,
                                                 mock_cluster_template,

diff --git a/releasenotes/notes/extra-network-interface-68bc169cb467a13e.yaml b/releasenotes/notes/extra-network-interface-68bc169cb467a13e.yaml
@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Support extra_network and extra_subnet labels to allow users to assign
+    extra network interface to cluster nodes, e.g. storage network. Also adds
+    possibility to set custom security group on network port.