Merge pull request #2886 from vmware/sl-22.1.6

AV-198224: Fixes for NSX-T migration script

python/avi/migrationtools/avi_migration_utils.py

@@ -139,10 +139,16 @@ def get_tenant_ref(self, name):
         tenant = 'admin'
         if name and name.startswith('/'):
             parts = name.split('/', 2)
-            tenant = parts[1]
-            if not parts[2]:
-                LOG.warning('Invalid tenant ref : %s', name)
-            name = parts[2]
+            # Max length of parts is 3 assuming tenant present. If no tenant, parse just the name
+            if len(parts) == 2:
+                # e.g /test-pool
+                name = parts[1]
+            else:
+                # e.g /admin/test-pool
+                tenant = parts[1]
+                if not parts[2]:
+                    LOG.warning('Invalid tenant ref : %s', name)
+                name = parts[2]
         elif name and '/' in name:
             parts = name.split('/')
             # Changed the index to get the tenant and name in case of

python/avi/migrationtools/config_patch.py

@@ -214,8 +214,12 @@ def apply_obj_patch(self, obj_type, obj, patch_data, avi_cfg):
     def deep_update(self, d, u):
         for k, v in u.items():
             if isinstance(v, collections.Mapping):
+                if len(v.keys()) == 0:
+                    d[k] = {}
                 d[k] = self.deep_update(d.get(k, {}), v)
             elif isinstance(v, list):
+                if len(v) == 0:
+                    d[k] = []
                 for index, i in enumerate(v):
                     if isinstance(i, collections.Mapping) and k in d and len(d[k]) >= index + 1:
                         d[k][index] = self.deep_update(d[k][index], i)

python/avi/migrationtools/nsxt_converter/nsxt_util.py

@@ -284,9 +284,19 @@ def wrapper(*args, **kwargs):
     @retry_with_backoff(retries=10, backoff_in_seconds=1)
     def call_api_with_retry(self, api, *args, **kwargs):
         time.sleep(0.1)
-        results = api(*args, **kwargs)
-        if results and "results" in vars(results):
-            results = results.to_dict().get("results", [])
+        results = []
+        api_resp = api(*args, **kwargs)
+        if api_resp and "results" in vars(api_resp):
+            results = api_resp.to_dict().get("results", [])
+
+            while 'cursor' in api_resp.to_dict() and api_resp.to_dict()['cursor']:
+                print(f"Cursor: {api_resp.to_dict()['cursor']}")
+                kwargs['cursor'] = str(api_resp.to_dict()['cursor'])
+                api_resp = api(*args, **kwargs)
+                if api_resp and "results" in vars(api_resp):
+                    for result in api_resp.to_dict().get("results", []):
+                        results.append(result)
+
         return results
 
     def nsx_cleanup(self):
@@ -1250,11 +1260,13 @@ def update_network_service_obj(self, network_obj, floating_ip):
              })
 
     def get_nsx_group_details(self,group_path):
+        ns_group_name = None
         domain_id = group_path.split('domains/')[1].split("/groups")[0]
         ns_group_id = group_path.split('groups/')[1]
         ns_groups_list = self.call_api_with_retry(self.nsx_api_client.infra.domains.Groups.list, domain_id)
         ns_group = [ns_g for ns_g in ns_groups_list if ns_g['id'] == ns_group_id]
-        ns_group_name = ns_group[0]['display_name']
+        if ns_group:
+            ns_group_name = ns_group[0]['display_name']
         ns_ip_addr = None
         ip_addr = self.call_api_with_retry(self.nsx_api_client.infra.domains.groups.members.IpAddresses.list, domain_id,
                                            ns_group_id)

python/avi/migrationtools/nsxt_converter/profile_converter.py

@@ -211,10 +211,11 @@ def convert(self, alb_config, nsx_lb_config, prefix, tenant):
                                                                                      conv_status['skipped']))
 
     def convert_http(self, alb_pr, lb_pr):
+        xff_enabled = True if lb_pr.get('x_forwarded_for') else False
         alb_pr['type'] = 'APPLICATION_PROFILE_TYPE_HTTP'
         alb_pr['http_profile'] = dict(
-            xff_enabled=lb_pr.get('xForwardedFor', False),
-            http_to_https=lb_pr.get('httpRedirectToHttps', False),
+            xff_enabled=xff_enabled,
+            http_to_https=lb_pr.get('http_redirect_to_https'),
             keepalive_timeout=lb_pr.get('idle_timeout'),
             # nsx-t config value in bytes, so converting from bytes to KB for ALB
             client_max_header_size=lb_pr.get('request_header_size') / 1024,

python/avi/migrationtools/nsxt_converter/test/test_migrationtool.py

@@ -12,6 +12,8 @@
 import yaml
 
 import subprocess
+from avi.migrationtools.nsxt_converter.test.conftest import option
+
 from avi.migrationtools.avi_migration_utils import get_count, set_update_count
 from avi.migrationtools.nsxt_converter.nsxt_converter import NsxtConverter
 
@@ -21,8 +23,8 @@
     import verify_controller_is_up, clean_reboot
 
 
-config_file = pytest.config.getoption('--config')
-output_file = pytest.config.getoption('--out')
+config_file = option.config
+output_file = option.out
 
 with open(config_file) as f:
     file_attribute = yaml.load(f, Loader=yaml.Loader)
@@ -55,7 +57,8 @@
                                        'patch.yaml')),
              traffic_state = 'enable',
              default_params_file = None,
-             cloud_tenant = 'admin'
+             cloud_tenant = 'admin',
+             skip_datapath_check = False,
         )
 
 if not os.path.exists(setup.get("output_file_path")):
@@ -90,8 +93,8 @@ def Nsxt_conv(nsxt_ip=None, nsxt_user=None, nsxt_password=None, ssh_root_passwor
                      ansible_filter_types=ansible_filter_types, vs_level_status=vs_level_status,
                      option=option, ansible=ansible, no_object_merge=object_merge_check,
                      vs_state=vs_state, vs_filter=vs_filter, segroup=segroup, patch=patch,
-                     traffic_state=traffic_state, default_params_file=default_params_file, cloud_tenant=cloud_tenant
-                     )
+                     traffic_state=traffic_state, default_params_file=default_params_file, cloud_tenant=cloud_tenant,
+                     skip_datapath_check = False)
     nsxt_converter = NsxtConverter(args)
     nsxt_converter.convert_lb_config(args)
 

python/avi/migrationtools/nsxt_converter/vs_converter.py

@@ -226,11 +226,23 @@ def convert(self, alb_config, nsx_lb_config, prefix, tenant, vs_state, controlle
                     else:
                         vsvip_ref = conv_utils.get_object_ref(vip["name"], 'vsvip', tenant=tenant, cloud_name=cloud_name)
                     alb_vs['vsvip_ref'] = vsvip_ref
-                alb_vs['services'] = [
-                    dict(
-                        port=int(lb_vs.get('ports')[0]),
-                        enable_ssl=False
-                    )]
+                alb_vs['services'] = []
+                ports = lb_vs.get('ports')
+                for port in ports:
+                    if isinstance(port, str) and '-' in port:
+                        port_start = port.split("-")[0]
+                        port_end_range = port.split("-")[-1]
+                        port_dict = dict(
+                            port=int(port_start),
+                            enable_ssl=False,
+                            port_range_end=port_end_range
+                        )
+                    else:
+                        port_dict = dict(
+                            port=int(port),
+                            enable_ssl=False
+                        )
+                    alb_vs['services'].append(port_dict)
                 skipped = [val for val in lb_vs.keys()
                            if val not in self.supported_attr]
                 na_list = [val for val in lb_vs.keys()
@@ -713,9 +725,6 @@ def convert(self, alb_config, nsx_lb_config, prefix, tenant, vs_state, controlle
                         main_pool_ref = sorry_pool_ref
                         is_pg_created = True
 
-                if not pool_present and self.skip_datapath_check:
-                    # As datapath validation is skipped, attach pool anyway
-                    pool_present = True
                 if is_pg_created:
                     self.add_teir_to_poolgroup(main_pool_ref, alb_config, tier1_lr)
                     self.update_poolgroup_with_cloud(main_pool_ref, alb_config, cloud_name, tenant, cloud_tenant)
@@ -829,9 +838,9 @@ def convert(self, alb_config, nsx_lb_config, prefix, tenant, vs_state, controlle
 
                 LOG.info('[VirtualService] Migration completed for VS {}'.format(lb_vs['display_name']))
             except Exception as e:
-                LOG.error("[VirtualService] Failed to convert VirtualService: {}".format(e))
+                LOG.error("[VirtualService] Error while converting VirtualService: {}".format(e))
                 update_count('error')
-                LOG.error("[VirtualServer] Failed to convert Monitor: %s" % lb_vs['display_name'],
+                LOG.error("[VirtualServer] Failed to convert VirtualService: %s" % lb_vs['display_name'],
                           exc_info=True)
                 conv_utils.add_status_row('virtualservice', None, lb_vs['display_name'],
                                           conv_const.STATUS_ERROR, "Conversion failure")