Add support for VMs with multiple IB devices

azurelinuxagent/common/rdma.py

@@ -199,7 +199,7 @@ class RDMADeviceHandler(object):
     ipoib_check_interval_sec = 1
 
     ipv4_addr = None
-    mac_adr = None
+    mac_addr = None
     nd_version = None
 
     def __init__(self, ipv4_addr, mac_addr, nd_version):
@@ -265,11 +265,151 @@ def provision_network_direct_rdma(self):
         RDMADeviceHandler.update_network_interface(self.mac_addr, self.ipv4_addr)
 
     def provision_sriov_rdma(self):  # pylint: disable=R1711
-        RDMADeviceHandler.wait_any_rdma_device(
-            self.sriov_dir, self.device_check_timeout_sec, self.device_check_interval_sec)
-        RDMADeviceHandler.update_iboip_interface(self.ipv4_addr, self.ipoib_check_timeout_sec,
-                                                 self.ipoib_check_interval_sec)
-        return
+
+        (key, value) = self.read_ipoib_data()
+        if key:
+            # provision multiple IP over IB addresses
+            logger.info("RDMA: provisioning multiple IP over IB addresses")
+            self.provision_sriov_multiple_ib(value)
+        elif self.ipv4_addr:
+            logger.info("RDMA: provisioning single IP over IB address")
+            # provision a single IP over IB address
+            RDMADeviceHandler.wait_any_rdma_device(self.sriov_dir,
+                self.device_check_timeout_sec, self.device_check_interval_sec)
+            RDMADeviceHandler.update_iboip_interface(self.ipv4_addr,
+                self.ipoib_check_timeout_sec, self.ipoib_check_interval_sec)
+        else:
+            logger.info("RDMA: missing IP address")
+
+    def read_ipoib_data(self) :
+
+        # read from KVP pool 0 to figure out the IP over IB addresses
+        kvp_key_size = 512
+        kvp_value_size = 2048
+        driver_info_source = '/var/lib/hyperv/.kvp_pool_0'
+
+        if not os.path.isfile(driver_info_source):
+            logger.error("RDMA: can't read KVP pool 0")
+            return (None, None)
+
+        key_0 = None
+        value_0 = None
+        with open(driver_info_source, "rb") as pool_file:
+            while True:
+                key = pool_file.read(kvp_key_size)
+                value = pool_file.read(kvp_value_size)
+                if key and value:
+                    key_0 = key.partition(b"\x00")[0]
+                    if key_0 :
+                        key_0 = key_0.decode()
+                    if key_0 == "IPoIB_Data":
+                        value_0 = value.partition(b"\x00")[0]
+                        if value_0 :
+                            value_0 = value_0.decode()
+                        break
+                else:
+                    break
+
+        if key_0 == "IPoIB_Data":
+            return (key_0, value_0)
+
+        return (None, None)
+
+    def provision_sriov_multiple_ib(self, value) :
+
+        mac_ip_array = []
+
+        values = value.split("|")
+        num_ips = len(values) - 1
+        # values[0] tells how many IPs. Format - NUMPAIRS:<number>
+        match = re.match(r"NUMPAIRS:(\d+)", values[0])
+        if match:
+            num = int(match.groups(0)[0])
+            if num != num_ips:
+                logger.error("RDMA: multiple IPs reported num={0} actual number of IPs={1}".format(num, num_ips))
+                return
+        else:
+            logger.error("RDMA: failed to find number of IP addresses in {0}".format(values[0]))
+            return
+
+        for i in range(1, num_ips+1):
+            # each MAC/IP entry is of format <MAC>:<IP>
+            match = re.match(r"([^:]+):(\d+\.\d+\.\d+\.\d+)", values[i])
+            if match:
+                mac_addr = match.groups(0)[0]
+                ipv4_addr = match.groups(0)[1]
+                mac_ip_array.append((mac_addr, ipv4_addr))
+            else:
+                logger.error("RDMA: failed to find MAC/IP address in {0}".format(values[i]))
+                return
+
+        # try to assign all MAC/IP addresses to IB interfaces
+        # retry for up to 60 times, with 1 seconds delay between each
+        retry = 60
+        while retry > 0:
+            count = self.update_iboip_interfaces(mac_ip_array)
+            if count == len(mac_ip_array):
+                return
+
+            time.sleep(1)
+            retry -= 1
+
+        logger.error("RDMA: failed to set all IP over IB addresses")
+
+    # Assign addresses to all IP over IB interfaces specified in mac_ip_array
+    # Return the number of IP addresses successfully assigned
+
+    def update_iboip_interfaces(self, mac_ip_array):
+
+        net_dir = "/sys/class/net"
+        nics = os.listdir(net_dir)
+        count = 0
+
+        for nic in nics:
+            # look for IBoIP interface of format ibXXX
+            if not re.match(r"ib\d+", nic):
+                continue
+
+            mac_addr = None
+            with open(os.path.join(net_dir, nic, "address")) as address_file:
+                mac_addr = address_file.read()
+
+            if not mac_addr:
+                logger.error("RDMA: can't read address for device {0}".format(nic))
+                continue
+
+            mac_addr = mac_addr.upper()
+
+            match = re.match(r".+(\w\w):(\w\w):(\w\w):\w\w:\w\w:(\w\w):(\w\w):(\w\w)\n", mac_addr)
+            if not match:
+                logger.error("RDMA: failed to parse address for device {0} address {1}".format(nic, mac_addr))
+                continue
+
+            # format an MAC address without :
+            mac_addr = ""
+            mac_addr = mac_addr.join(match.groups(0))
+
+            for mac_ip in mac_ip_array:
+                if mac_ip[0] == mac_addr:
+                    ret = 0
+                    try:
+                        ip_command = ["ip", "addr", "add", "{0}/16".format(mac_ip[1]), "dev", nic]
+                        shellutil.run_command(ip_command)
+                    except shellutil.CommandError as error:
+                        ret = error.returncode
+
+                    if ret == 0:
+                        logger.info("RDMA: set address {0} to device {1}".format(mac_ip[1], nic))
+
+                    if ret and ret != 2:
+                        # return value 2 means the address is already set
+                        logger.error("RDMA: failed to set IP address {0} on device {1}".format(mac_ip[1], nic))
+                    else:
+                        count += 1
+
+                    break
+
+        return count
 
     @staticmethod
     def update_iboip_interface(ipv4_addr, timeout_sec, check_interval_sec):