Port 202012 DB migration changes to newer branches (sonic-net#2515)

Fixes sonic-net/sonic-buildimage#12643 Version on 202012 branch is missing in master and 202205 branch. This caused failures in upgrade from 202012 to newer branch images, where db_migrator experienced crash. This commit fixes the issue, and ports the version handling for 202012 to newer branches.
mdanish-kh · Nov 23, 2022 · aa7a3a3 · aa7a3a3
1 parent 3c8b7d7
commit aa7a3a3
Show file tree

Hide file tree

Showing 194 changed files with 422 additions and 72 deletions.
diff --git a/scripts/db_migrator.py b/scripts/db_migrator.py
@@ -8,7 +8,9 @@
 import re
 
 from sonic_py_common import device_info, logger
+from swsssdk import ConfigDBConnector, SonicDBConfig
 from swsscommon.swsscommon import SonicV2Connector, ConfigDBConnector, SonicDBConfig
+from db_migrator_constants import RESTAPI, TELEMETRY, CONSOLE_SWITCH
 
 INIT_CFG_FILE = '/etc/sonic/init_cfg.json'
 
@@ -489,6 +491,61 @@ def migrate_qos_fieldval_reference_format(self):
         self.migrate_qos_db_fieldval_reference_remove(qos_table_list, self.configDB, self.configDB.CONFIG_DB, '|')
         return True
 
+    def migrate_vxlan_config(self):
+        log.log_notice('Migrate VXLAN table config')
+        # Collect VXLAN data from config DB
+        vxlan_data = self.configDB.keys(self.configDB.CONFIG_DB, "VXLAN_TUNNEL*")
+        if not vxlan_data:
+            # do nothing if vxlan entries are not present in configdb
+            return
+        for vxlan_table in vxlan_data:
+            vxlan_map_mapping = self.configDB.get_all(self.configDB.CONFIG_DB, vxlan_table)
+            tunnel_keys = vxlan_table.split(self.configDB.KEY_SEPARATOR)
+            tunnel_keys[0] = tunnel_keys[0] + "_TABLE"
+            vxlan_table = self.appDB.get_db_separator(self.appDB.APPL_DB).join(tunnel_keys)
+            for field, value in vxlan_map_mapping.items():
+                # add entries from configdb to appdb only when they are missing
+                if not self.appDB.hexists(self.appDB.APPL_DB, vxlan_table, field):
+                    log.log_notice('Copying vxlan entries from configdb to appdb: updated {} with {}:{}'.format(
+                        vxlan_table, field, value))
+                    self.appDB.set(self.appDB.APPL_DB, vxlan_table, field, value)
+
+    def migrate_restapi(self):
+        # RESTAPI - add missing key
+        log.log_notice('Migrate RESTAPI configuration')
+        config = self.configDB.get_entry('RESTAPI', 'config')
+        if not config:
+            self.configDB.set_entry("RESTAPI", "config", RESTAPI.get("config"))
+        certs = self.configDB.get_entry('RESTAPI', 'certs')
+        if not certs:
+            self.configDB.set_entry("RESTAPI", "certs", RESTAPI.get("certs"))
+
+    def migrate_telemetry(self):
+        # TELEMETRY - add missing key
+        log.log_notice('Migrate TELEMETRY configuration')
+        gnmi = self.configDB.get_entry('TELEMETRY', 'gnmi')
+        if not gnmi:
+            self.configDB.set_entry("TELEMETRY", "gnmi", TELEMETRY.get("gnmi"))
+        certs = self.configDB.get_entry('TELEMETRY', 'certs')
+        if not certs:
+            self.configDB.set_entry("TELEMETRY", "certs", TELEMETRY.get("certs"))
+
+    def migrate_console_switch(self):
+        # CONSOLE_SWITCH - add missing key
+        log.log_notice('Migrate CONSOLE_SWITCH configuration')
+        console_mgmt = self.configDB.get_entry('CONSOLE_SWITCH', 'console_mgmt')
+        if not console_mgmt:
+            self.configDB.set_entry("CONSOLE_SWITCH", "console_mgmt",
+                CONSOLE_SWITCH.get("console_mgmt"))
+
+    def migrate_device_metadata(self):
+        # DEVICE_METADATA - synchronous_mode entry
+        log.log_notice('Migrate DEVICE_METADATA missing configuration (synchronous_mode=enable)')
+        metadata = self.configDB.get_entry('DEVICE_METADATA', 'localhost')
+        if 'synchronous_mode' not in metadata:
+            metadata['synchronous_mode'] = 'enable'
+            self.configDB.set_entry('DEVICE_METADATA', 'localhost', metadata)
+
     def migrate_port_qos_map_global(self):
         """
         Generate dscp_to_tc_map for switch.
@@ -651,10 +708,25 @@ def version_2_0_0(self):
 
     def version_2_0_1(self):
         """
-        Version 2_0_1.
-        This is the latest version for 202012 branch 
+        Handle and migrate missing config that results from cross branch upgrade to
+        202012 as target.
         """
         log.log_info('Handling version_2_0_1')
+        self.migrate_vxlan_config()
+        self.migrate_restapi()
+        self.migrate_telemetry()
+        self.migrate_console_switch()
+        self.migrate_device_metadata()
+
+        self.set_version('version_2_0_2')
+        return 'version_2_0_2'
+
+    def version_2_0_2(self):
+        """
+        Version 2_0_2
+        This is the latest version for 202012 branch 
+        """
+        log.log_info('Handling version_2_0_2')
         self.set_version('version_3_0_0')
         return 'version_3_0_0'
 

diff --git a/scripts/db_migrator_constants.py b/scripts/db_migrator_constants.py
@@ -0,0 +1,32 @@
+RESTAPI = {
+        "config": {
+            "client_auth": "true", 
+            "log_level": "info", 
+            "allow_insecure": "false"
+        }, 
+        "certs": {
+            "server_key": "/etc/sonic/credentials/restapiserver.key", 
+            "ca_crt": "/etc/sonic/credentials/AME_ROOT_CERTIFICATE.pem", 
+            "server_crt": "/etc/sonic/credentials/restapiserver.crt", 
+            "client_crt_cname": "client.restapi.sonic.gbl"
+        }
+    }
+
+TELEMETRY = {
+    "gnmi": {
+        "client_auth": "true", 
+        "log_level": "2", 
+        "port": "50051"
+    }, 
+    "certs": {
+        "server_key": "/etc/sonic/telemetry/streamingtelemetryserver.key", 
+        "ca_crt": "/etc/sonic/telemetry/dsmsroot.cer", 
+        "server_crt": "/etc/sonic/telemetry/streamingtelemetryserver.cer"
+    }
+}
+
+CONSOLE_SWITCH = {
+    "console_mgmt": {
+        "enabled": "no"
+    }
+}
diff --git a/tests/db_migrator_input/config_db/acs-msn2700-t0-version_1_0_1.json b/tests/db_migrator_input/config_db/acs-msn2700-t0-version_1_0_1.json
@@ -731,6 +731,7 @@
         "hwsku": "ACS-MSN2700",
         "default_bgp_status": "up",
         "type": "ToRRouter",
+        "synchronous_mode": "enable",
         "hostname": "sonic",
         "platform": "x86_64-mlnx_msn2700-r0",
         "mac": "00:01:02:03:04:00",

diff --git a/tests/db_migrator_input/config_db/acs-msn2700-t0-version_1_0_2.json b/tests/db_migrator_input/config_db/acs-msn2700-t0-version_1_0_2.json
@@ -737,7 +737,8 @@
         "default_pfcwd_status": "disable",
         "bgp_asn": "65100",
         "deployment_id": "1",
-        "type": "ToRRouter"
+        "type": "ToRRouter",
+        "synchronous_mode": "enable"
     },
     "PORT|Ethernet0": {
         "lanes": "0,1,2,3",

diff --git a/tests/db_migrator_input/config_db/acs-msn2700-t0-version_1_0_3.json b/tests/db_migrator_input/config_db/acs-msn2700-t0-version_1_0_3.json
@@ -737,7 +737,8 @@
         "default_pfcwd_status": "disable",
         "bgp_asn": "65100",
         "deployment_id": "1",
-        "type": "ToRRouter"
+        "type": "ToRRouter",
+        "synchronous_mode": "enable"
     },
     "PORT|Ethernet0": {
         "lanes": "0,1,2,3",

diff --git a/tests/db_migrator_input/config_db/acs-msn2700-t0-version_1_0_4.json b/tests/db_migrator_input/config_db/acs-msn2700-t0-version_1_0_4.json
@@ -737,7 +737,8 @@
         "default_pfcwd_status": "disable",
         "bgp_asn": "65100",
         "deployment_id": "1",
-        "type": "ToRRouter"
+        "type": "ToRRouter",
+        "synchronous_mode": "enable"
     },
     "PORT|Ethernet0": {
         "lanes": "0,1,2,3",

diff --git a/tests/db_migrator_input/config_db/acs-msn2700-t0-version_1_0_5.json b/tests/db_migrator_input/config_db/acs-msn2700-t0-version_1_0_5.json
@@ -737,7 +737,8 @@
         "default_pfcwd_status": "disable",
         "bgp_asn": "65100",
         "deployment_id": "1",
-        "type": "ToRRouter"
+        "type": "ToRRouter",
+        "synchronous_mode": "enable"
     },
     "PORT|Ethernet0": {
         "lanes": "0,1,2,3",

diff --git a/tests/db_migrator_input/config_db/acs-msn2700-t0-version_1_0_6.json b/tests/db_migrator_input/config_db/acs-msn2700-t0-version_1_0_6.json
@@ -737,7 +737,8 @@
         "default_pfcwd_status": "disable",
         "bgp_asn": "65100",
         "deployment_id": "1",
-        "type": "ToRRouter"
+        "type": "ToRRouter",
+        "synchronous_mode": "enable"
     },
     "PORT|Ethernet0": {
         "lanes": "0,1,2,3",

diff --git a/tests/db_migrator_input/config_db/acs-msn2700-t0-version_3_0_0.json b/tests/db_migrator_input/config_db/acs-msn2700-t0-version_3_0_0.json
@@ -689,7 +689,8 @@
         "bgp_asn": "65100",
         "buffer_model": "dynamic",
         "deployment_id": "1",
-        "type": "ToRRouter"
+        "type": "ToRRouter",
+        "synchronous_mode": "enable"
     },
     "LOSSLESS_TRAFFIC_PATTERN|AZURE": {
         "small_packet_percentage": "100",

diff --git a/tests/db_migrator_input/config_db/acs-msn2700-t0-version_3_0_3.json b/tests/db_migrator_input/config_db/acs-msn2700-t0-version_3_0_3.json
@@ -689,7 +689,8 @@
         "bgp_asn": "65100",
         "buffer_model": "dynamic",
         "deployment_id": "1",
-        "type": "ToRRouter"
+        "type": "ToRRouter",
+        "synchronous_mode": "enable"
     },
     "LOSSLESS_TRAFFIC_PATTERN|AZURE": {
         "small_packet_percentage": "100",
@@ -1016,4 +1017,4 @@
     "VERSIONS|DATABASE": {
         "VERSION": "version_3_0_3"
     }
-}
+}
diff --git a/tests/db_migrator_input/config_db/acs-msn2700-t1-version_1_0_1.json b/tests/db_migrator_input/config_db/acs-msn2700-t1-version_1_0_1.json
@@ -824,6 +824,7 @@
         "hwsku": "ACS-MSN2700",
         "default_bgp_status": "up",
         "type": "LeafRouter",
+        "synchronous_mode": "enable",
         "hostname": "sonic",
         "platform": "x86_64-mlnx_msn2700-r0",
         "mac": "00:01:02:03:04:00",

diff --git a/tests/db_migrator_input/config_db/acs-msn2700-t1-version_1_0_2.json b/tests/db_migrator_input/config_db/acs-msn2700-t1-version_1_0_2.json
@@ -824,6 +824,7 @@
         "hwsku": "ACS-MSN2700",
         "default_bgp_status": "up",
         "type": "LeafRouter",
+        "synchronous_mode": "enable",
         "hostname": "sonic",
         "platform": "x86_64-mlnx_msn2700-r0",
         "mac": "00:01:02:03:04:00",

diff --git a/tests/db_migrator_input/config_db/acs-msn2700-t1-version_1_0_3.json b/tests/db_migrator_input/config_db/acs-msn2700-t1-version_1_0_3.json
@@ -824,6 +824,7 @@
         "hwsku": "ACS-MSN2700",
         "default_bgp_status": "up",
         "type": "LeafRouter",
+        "synchronous_mode": "enable",
         "hostname": "sonic",
         "platform": "x86_64-mlnx_msn2700-r0",
         "mac": "00:01:02:03:04:00",

diff --git a/tests/db_migrator_input/config_db/acs-msn2700-t1-version_1_0_4.json b/tests/db_migrator_input/config_db/acs-msn2700-t1-version_1_0_4.json
@@ -824,6 +824,7 @@
         "hwsku": "ACS-MSN2700",
         "default_bgp_status": "up",
         "type": "LeafRouter",
+        "synchronous_mode": "enable",
         "hostname": "sonic",
         "platform": "x86_64-mlnx_msn2700-r0",
         "mac": "00:01:02:03:04:00",

diff --git a/tests/db_migrator_input/config_db/acs-msn2700-t1-version_1_0_5.json b/tests/db_migrator_input/config_db/acs-msn2700-t1-version_1_0_5.json
@@ -824,6 +824,7 @@
         "hwsku": "ACS-MSN2700",
         "default_bgp_status": "up",
         "type": "LeafRouter",
+        "synchronous_mode": "enable",
         "hostname": "sonic",
         "platform": "x86_64-mlnx_msn2700-r0",
         "mac": "00:01:02:03:04:00",

diff --git a/tests/db_migrator_input/config_db/acs-msn2700-t1-version_1_0_6.json b/tests/db_migrator_input/config_db/acs-msn2700-t1-version_1_0_6.json
@@ -824,6 +824,7 @@
         "hwsku": "ACS-MSN2700",
         "default_bgp_status": "up",
         "type": "LeafRouter",
+        "synchronous_mode": "enable",
         "hostname": "sonic",
         "platform": "x86_64-mlnx_msn2700-r0",
         "mac": "00:01:02:03:04:00",

diff --git a/tests/db_migrator_input/config_db/acs-msn2700-t1-version_3_0_0.json b/tests/db_migrator_input/config_db/acs-msn2700-t1-version_3_0_0.json
@@ -754,6 +754,7 @@
         "hwsku": "ACS-MSN2700",
         "default_bgp_status": "up",
         "type": "LeafRouter",
+        "synchronous_mode": "enable",
         "hostname": "sonic",
         "platform": "x86_64-mlnx_msn2700-r0",
         "mac": "00:01:02:03:04:00",

diff --git a/tests/db_migrator_input/config_db/acs-msn2700-t1-version_3_0_3.json b/tests/db_migrator_input/config_db/acs-msn2700-t1-version_3_0_3.json
@@ -761,7 +761,8 @@
         "bgp_asn": "65100",
         "buffer_model": "dynamic",
         "deployment_id": "1",
-        "docker_routing_config_mode": "unified"
+        "docker_routing_config_mode": "unified",
+        "synchronous_mode": "enable"
     },
     "LOSSLESS_TRAFFIC_PATTERN|AZURE": {
         "small_packet_percentage": "100",
@@ -1088,4 +1089,4 @@
     "VERSIONS|DATABASE": {
         "VERSION": "version_3_0_3"
     }
-}
+}
diff --git a/tests/db_migrator_input/config_db/acs-msn3700-t0-version_1_0_2.json b/tests/db_migrator_input/config_db/acs-msn3700-t0-version_1_0_2.json
@@ -738,6 +738,7 @@
         "hwsku": "ACS-MSN3700",
         "default_bgp_status": "up",
         "type": "ToRRouter",
+        "synchronous_mode": "enable",
         "region": "None",
         "hostname": "sonic",
         "platform": "x86_64-mlnx_msn3700-r0",

diff --git a/tests/db_migrator_input/config_db/acs-msn3700-t0-version_1_0_3.json b/tests/db_migrator_input/config_db/acs-msn3700-t0-version_1_0_3.json
@@ -864,6 +864,7 @@
         "hwsku": "ACS-MSN3700",
         "default_bgp_status": "up",
         "type": "ToRRouter",
+        "synchronous_mode": "enable",
         "region": "None",
         "hostname": "sonic",
         "platform": "x86_64-mlnx_msn3700-r0",

diff --git a/tests/db_migrator_input/config_db/acs-msn3700-t0-version_1_0_4.json b/tests/db_migrator_input/config_db/acs-msn3700-t0-version_1_0_4.json
@@ -864,6 +864,7 @@
         "hwsku": "ACS-MSN3700",
         "default_bgp_status": "up",
         "type": "ToRRouter",
+        "synchronous_mode": "enable",
         "region": "None",
         "hostname": "sonic",
         "platform": "x86_64-mlnx_msn3700-r0",

diff --git a/tests/db_migrator_input/config_db/acs-msn3700-t0-version_1_0_5.json b/tests/db_migrator_input/config_db/acs-msn3700-t0-version_1_0_5.json
@@ -864,6 +864,7 @@
         "hwsku": "ACS-MSN3700",
         "default_bgp_status": "up",
         "type": "ToRRouter",
+        "synchronous_mode": "enable",
         "region": "None",
         "hostname": "sonic",
         "platform": "x86_64-mlnx_msn3700-r0",

diff --git a/tests/db_migrator_input/config_db/acs-msn3700-t0-version_1_0_6.json b/tests/db_migrator_input/config_db/acs-msn3700-t0-version_1_0_6.json
@@ -864,6 +864,7 @@
         "hwsku": "ACS-MSN3700",
         "default_bgp_status": "up",
         "type": "ToRRouter",
+        "synchronous_mode": "enable",
         "region": "None",
         "hostname": "sonic",
         "platform": "x86_64-mlnx_msn3700-r0",

diff --git a/tests/db_migrator_input/config_db/acs-msn3700-t0-version_3_0_0.json b/tests/db_migrator_input/config_db/acs-msn3700-t0-version_3_0_0.json
@@ -808,6 +808,7 @@
         "hwsku": "ACS-MSN3700",
         "default_bgp_status": "up",
         "type": "ToRRouter",
+        "synchronous_mode": "enable",
         "region": "None",
         "hostname": "sonic",
         "platform": "x86_64-mlnx_msn3700-r0",

diff --git a/tests/db_migrator_input/config_db/acs-msn3700-t0-version_3_0_3.json b/tests/db_migrator_input/config_db/acs-msn3700-t0-version_3_0_3.json
@@ -817,7 +817,8 @@
         "buffer_model": "dynamic",
         "deployment_id": "1",
         "docker_routing_config_mode": "separated",
-        "cloudtype": "None"
+        "cloudtype": "None",
+        "synchronous_mode": "enable"
     },
     "LOSSLESS_TRAFFIC_PATTERN|AZURE": {
         "small_packet_percentage": "100",
@@ -1902,4 +1903,4 @@
     "VERSIONS|DATABASE": {
         "VERSION": "version_3_0_3"
     }
-}
+}
diff --git a/tests/db_migrator_input/config_db/acs-msn3700-t1-version_1_0_2.json b/tests/db_migrator_input/config_db/acs-msn3700-t1-version_1_0_2.json
@@ -838,6 +838,7 @@
         "hwsku": "ACS-MSN3700",
         "default_bgp_status": "up",
         "type": "LeafRouter",
+        "synchronous_mode": "enable",
         "region": "None",
         "hostname": "sonic",
         "platform": "x86_64-mlnx_msn3700-r0",

diff --git a/tests/db_migrator_input/config_db/acs-msn3700-t1-version_1_0_3.json b/tests/db_migrator_input/config_db/acs-msn3700-t1-version_1_0_3.json
@@ -940,6 +940,7 @@
         "hwsku": "ACS-MSN3700",
         "default_bgp_status": "up",
         "type": "LeafRouter",
+        "synchronous_mode": "enable",
         "region": "None",
         "hostname": "sonic",
         "platform": "x86_64-mlnx_msn3700-r0",

diff --git a/tests/db_migrator_input/config_db/acs-msn3700-t1-version_1_0_4.json b/tests/db_migrator_input/config_db/acs-msn3700-t1-version_1_0_4.json
@@ -940,6 +940,7 @@
         "hwsku": "ACS-MSN3700",
         "default_bgp_status": "up",
         "type": "LeafRouter",
+        "synchronous_mode": "enable",
         "region": "None",
         "hostname": "sonic",
         "platform": "x86_64-mlnx_msn3700-r0",

diff --git a/tests/db_migrator_input/config_db/acs-msn3700-t1-version_1_0_5.json b/tests/db_migrator_input/config_db/acs-msn3700-t1-version_1_0_5.json
@@ -940,6 +940,7 @@
         "hwsku": "ACS-MSN3700",
         "default_bgp_status": "up",
         "type": "LeafRouter",
+        "synchronous_mode": "enable",
         "region": "None",
         "hostname": "sonic",
         "platform": "x86_64-mlnx_msn3700-r0",

diff --git a/tests/db_migrator_input/config_db/acs-msn3700-t1-version_1_0_6.json b/tests/db_migrator_input/config_db/acs-msn3700-t1-version_1_0_6.json
@@ -940,6 +940,7 @@
         "hwsku": "ACS-MSN3700",
         "default_bgp_status": "up",
         "type": "LeafRouter",
+        "synchronous_mode": "enable",
         "region": "None",
         "hostname": "sonic",
         "platform": "x86_64-mlnx_msn3700-r0",