Support extra_network and extra_subnet labels

This allows users to add extra network interface to cluster nodes, e.g. storage network. Story: 2002448 Task: 21983 Co-Authored-By: Bharat Kunwar <bharat@stackhpc.com> Change-Id: I10a6a4d72e9ec635f2c73d9fe64a8d136228f532 (cherry picked from commit bea8c72) (cherry picked from commit 8260f11) (cherry picked from commit d6c0a2d) (cherry picked from commit 6227555)
stackhpc · Mar 28, 2024 · 3667d28 · 3667d28
1 parent dbd488c
commit 3667d28
Show file tree

Hide file tree

Showing 7 changed files with 195 additions and 7 deletions.
diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst
@@ -484,6 +484,12 @@ the table are linked to more details elsewhere in the user guide.
 +---------------------------------------+--------------------+---------------+
 | `octavia_lb_healthcheck`_             | see bellow         | true          |
 +---------------------------------------+--------------------+---------------+
+| `extra_network`_                      | see below          | ""            |
++---------------------------------------+--------------------+---------------+
+| `extra_subnet`_                       | see below          | ""            |
++---------------------------------------+--------------------+---------------+
+| `extra_security_group`_               | see below          | see below     |
++---------------------------------------+--------------------+---------------+
 
 .. _cluster:
 
@@ -1617,6 +1623,22 @@ _`octavia_lb_healthcheck`
   If true, enable Octavia load balancer healthcheck
   Default: true
 
+_`extra_network`
+  Optional additional network name or UUID to add to cluster nodes.
+  When not specified, additional networks are not added. Optionally specify
+  'extra_subnet' if you wish to use a specific subnet on the network.
+  Default: ""
+
+_`extra_subnet`
+  Optional additional subnet name or UUID to add to cluster nodes.
+  Only used when 'extra_network' is defined.
+  Default: ""
+
+_`extra_security_group`
+  Optional additional group name or UUID to add to network port.
+  Only used when 'extra_network' is defined.
+  Default: cluster node default security group.
+
 Supported versions
 ------------------
 
@@ -2463,7 +2485,6 @@ _`calico_tag`
   Victoria default: v3.13.1
   Wallaby default: v3.13.1
 
-
 Besides, the Calico network driver needs kube_tag with v1.9.3 or later, because
 Calico needs extra mounts for the kubelet container. See `commit
 <https://github.com/projectatomic/atomic-system-containers/commit/54ab8abc7fa1bfb6fa674f55cd0c2fa0c812fd36>`_

diff --git a/magnum/drivers/heat/k8s_fedora_template_def.py b/magnum/drivers/heat/k8s_fedora_template_def.py
@@ -127,7 +127,8 @@ def get_params(self, context, cluster_template, cluster, **kwargs):
                       'min_node_count', 'max_node_count', 'npd_enabled',
                       'ostree_remote', 'ostree_commit',
                       'use_podman', 'kube_image_digest',
-                      'metrics_scraper_tag']
+                      'metrics_scraper_tag',
+                      'extra_network', 'extra_subnet', 'extra_security_group']
 
         labels = self._get_relevant_labels(cluster, kwargs)
 

diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml
@@ -1008,13 +1008,32 @@ parameters:
     description: >
       Post install manifest URL used to setup some cloud provider/vendor
       specific configs
-    default: ""
+    default: ''
 
   master_lb_allowed_cidrs:
     type: comma_delimited_list
     description: The allowed CIDR list for master load balancer
     default: []
 
+  extra_network:
+    type: string
+    description: >
+      Additional network to bind nodes to
+    default: ''
+
+  extra_subnet:
+    type: string
+    description: >
+      Subnet for additional network
+    default: ''
+
+  extra_security_group:
+    type: string
+    description: >
+      Additional security group name
+    default: ''
+
+
 resources:
 
   ######################################################################
@@ -1407,6 +1426,9 @@ resources:
           containerd_tarball_sha256: {get_param: containerd_tarball_sha256}
           post_install_manifest_url: {get_param: post_install_manifest_url}
           metrics_scraper_tag: {get_param: metrics_scraper_tag}
+          extra_network: {get_param: extra_network}
+          extra_subnet: {get_param: extra_subnet}
+          extra_security_group: {get_param: extra_security_group}
 
   kube_cluster_config:
     condition: create_cluster_resources
@@ -1589,6 +1611,10 @@ resources:
           containerd_tarball_sha256: {get_param: containerd_tarball_sha256}
           kube_service_account_key: {get_param: kube_service_account_key}
           kube_service_account_private_key: {get_param: kube_service_account_private_key}
+          extra_network: {get_param: extra_network}
+          extra_subnet: {get_param: extra_subnet}
+          extra_security_group: {get_param: extra_security_group}
+
 outputs:
 
   api_address:

diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml
@@ -714,6 +714,21 @@ parameters:
     description: >
       Tag of metrics-scraper for kubernetes dashboard.
 
+  extra_network:
+    type: string
+    description: >
+      Additional network name to bind ports to instances
+
+  extra_subnet:
+    type: string
+    description: >
+      Additional subnet name
+
+  extra_security_group:
+    type: string
+    description: >
+      Additional security group name
+
 conditions:
 
   image_based: {equals: [{get_param: boot_volume_size}, 0]}
@@ -723,6 +738,24 @@ conditions:
       - get_param: boot_volume_size
       - 0
 
+  extra_network_defined:
+    not:
+      equals:
+      - get_param: extra_network
+      - ''
+
+  extra_subnet_defined:
+    not:
+      equals:
+      - get_param: extra_subnet
+      - ''
+
+  extra_security_group_defined:
+    not:
+      equals:
+      - get_param: extra_security_group
+      - ''
+
 resources:
   ######################################################################
   #
@@ -985,7 +1018,12 @@ resources:
       software_config_transport: POLL_SERVER_HEAT
       user_data: {get_resource: agent_config}
       networks:
-        - port: {get_resource: kube_master_eth0}
+        list_concat:
+          - - port: {get_resource: kube_master_eth0}
+          - if:
+              - "extra_network_defined"
+              - - port: {get_resource: kube_master_eth1}
+              - []
       scheduler_hints: { group: { get_param: nodes_server_group_id }}
       availability_zone: {get_param: availability_zone}
 
@@ -999,7 +1037,12 @@ resources:
       software_config_transport: POLL_SERVER_HEAT
       user_data: {get_resource: agent_config}
       networks:
-        - port: {get_resource: kube_master_eth0}
+        list_concat:
+          - - port: {get_resource: kube_master_eth0}
+          - if:
+              - "extra_network_defined"
+              - - port: {get_resource: kube_master_eth1}
+              - []
       scheduler_hints: { group: { get_param: nodes_server_group_id }}
       availability_zone: {get_param: availability_zone}
       block_device_mapping_v2:
@@ -1018,6 +1061,23 @@ resources:
         - ip_address: {get_param: pods_network_cidr}
       replacement_policy: AUTO
 
+  kube_master_eth1:
+    type: OS::Neutron::Port
+    condition: extra_network_defined
+    properties:
+      network: {get_param: extra_network}
+      security_groups:
+        - if:
+            - "extra_security_group_defined"
+            - {get_param: extra_security_group}
+            - {get_param: secgroup_kube_master_id}
+      fixed_ips:
+        if:
+          - "extra_subnet_defined"
+          - - subnet: {get_param: extra_subnet}
+          - []
+      replacement_policy: AUTO
+
   kube_master_floating:
     type: Magnum::Optional::KubeMaster::Neutron::FloatingIP
     properties:

diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubeminion.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubeminion.yaml
@@ -387,6 +387,21 @@ parameters:
       The private key will be used to sign generated k8s service account
       tokens.
 
+  extra_network:
+    type: string
+    description: >
+      Additional network name to bind ports to instances
+
+  extra_subnet:
+    type: string
+    description: >
+      Additional subnet name
+
+  extra_security_group:
+    type: string
+    description: >
+      Additional seurity group name
+
 conditions:
 
   image_based: {equals: [{get_param: boot_volume_size}, 0]}
@@ -396,6 +411,25 @@ conditions:
       - get_param: boot_volume_size
       - 0
 
+  extra_network_defined:
+    not:
+      equals:
+      - get_param: extra_network
+      - ''
+
+  extra_subnet_defined:
+    not:
+      equals:
+      - get_param: extra_subnet
+      - ''
+
+  extra_security_group_defined:
+    not:
+      equals:
+      - get_param: extra_security_group
+      - ''
+
+
 resources:
 
   agent_config:
@@ -563,7 +597,12 @@ resources:
       user_data_format: SOFTWARE_CONFIG
       software_config_transport: POLL_SERVER_HEAT
       networks:
-        - port: {get_resource: kube_minion_eth0}
+        list_concat:
+          - - port: {get_resource: kube_minion_eth0}
+          - if:
+              - "extra_network_defined"
+              - - port: {get_resource: kube_minion_eth1}
+              - []
       scheduler_hints: { group: { get_param: nodes_server_group_id }}
       availability_zone: {get_param: availability_zone}
 
@@ -577,7 +616,12 @@ resources:
       user_data_format: SOFTWARE_CONFIG
       software_config_transport: POLL_SERVER_HEAT
       networks:
-        - port: {get_resource: kube_minion_eth0}
+        list_concat:
+          - - port: {get_resource: kube_minion_eth0}
+          - if:
+              - "extra_network_defined"
+              - - port: {get_resource: kube_minion_eth1}
+              - []
       scheduler_hints: { group: { get_param: nodes_server_group_id }}
       availability_zone: {get_param: availability_zone}
       block_device_mapping_v2:
@@ -596,6 +640,23 @@ resources:
         - ip_address: {get_param: pods_network_cidr}
       replacement_policy: AUTO
 
+  kube_minion_eth1:
+    type: OS::Neutron::Port
+    condition: extra_network_defined
+    properties:
+      network: {get_param: extra_network}
+      security_groups:
+        - if:
+            - "extra_security_group_defined"
+            - get_param: extra_security_group
+            - get_param: secgroup_kube_minion_id
+      fixed_ips:
+        if:
+          - "extra_subnet_defined"
+          - - subnet: {get_param: extra_subnet}
+          - []
+      replacement_policy: AUTO
+
   kube_minion_floating:
     type: Magnum::Optional::KubeMinion::Neutron::FloatingIP
     properties:

diff --git a/magnum/tests/unit/drivers/test_template_definition.py b/magnum/tests/unit/drivers/test_template_definition.py
@@ -615,6 +615,9 @@ def test_k8s_get_params(self, mock_generate_csr_and_key,
         octavia_lb_algorithm = mock_cluster.labels.get('octavia_lb_algorithm')
         octavia_lb_healthcheck = mock_cluster.labels.get(
                 'octavia_lb_healthcheck')
+        extra_network = mock_cluster.labels.get('extra_network')
+        extra_subnet = mock_cluster.labels.get('extra_subnet')
+        extra_security_group = mock_cluster.labels.get('extra_security_group')
 
         k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
 
@@ -742,6 +745,9 @@ def test_k8s_get_params(self, mock_generate_csr_and_key,
             'octavia_provider': octavia_provider,
             'octavia_lb_algorithm': octavia_lb_algorithm,
             'octavia_lb_healthcheck': octavia_lb_healthcheck,
+            'extra_network': extra_network,
+            'extra_subnet': extra_subnet,
+            'extra_security_group': extra_security_group,
         }}
         mock_get_params.assert_called_once_with(mock_context,
                                                 mock_cluster_template,
@@ -1175,6 +1181,10 @@ def test_k8s_get_params_insecure(self, mock_generate_csr_and_key,
 
         master_lb_allowed_cidrs = mock_cluster.labels.get(
             'master_lb_allowed_cidrs')
+        extra_network = mock_cluster.labels.get('extra_network')
+        extra_subnet = mock_cluster.labels.get('extra_subnet')
+        extra_security_group = mock_cluster.labels.get('extra_security_group')
+
 
         octavia_provider = mock_cluster.labels.get('octavia_provider')
         octavia_lb_algorithm = mock_cluster.labels.get('octavia_lb_algorithm')
@@ -1309,6 +1319,9 @@ def test_k8s_get_params_insecure(self, mock_generate_csr_and_key,
             'octavia_provider': octavia_provider,
             'octavia_lb_algorithm': octavia_lb_algorithm,
             'octavia_lb_healthcheck': octavia_lb_healthcheck,
+            'extra_network': extra_network,
+            'extra_subnet': extra_subnet,
+            'extra_security_group': extra_security_group,
         }}
         mock_get_params.assert_called_once_with(mock_context,
                                                 mock_cluster_template,

diff --git a/releasenotes/notes/extra-network-interface-68bc169cb467a13e.yaml b/releasenotes/notes/extra-network-interface-68bc169cb467a13e.yaml
@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Support extra_network and extra_subnet labels to allow users to assign
+    extra network interface to cluster nodes, e.g. storage network. Also adds
+    possibility to set custom security group on network port.