show commands for SYSTEM READY (#1851)

What I did
show command support for System Ready Feature Implementation .
Also sonic-package-manager support for app readiness.

"show system-health sysready-status"
"show system-health sysready-status detail"
"show system-health sysready-status brief"

How I did it
Introduce "show system-health sysready-status" click CLI with options.

How to verify it
Check the click CLIs introduced.

scripts/sysreadyshow

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+
+"""
+    Script to show system ready status.
+"""
+
+import os
+import sys
+import argparse
+from tabulate import tabulate
+from natsort import natsorted
+
+# mock the redis for unit test purposes #
+try:
+    if os.environ["UTILITIES_UNIT_TESTING"] == "1":
+        modules_path = os.path.join(os.path.dirname(__file__), "..")
+        test_path = os.path.join(modules_path, "tests")
+        sys.path.insert(0, modules_path)
+        sys.path.insert(0, test_path)
+        import mock_tables.dbconnector #lgtm [py/unused-import]
+except KeyError:
+    pass
+
+from swsscommon.swsscommon import SonicV2Connector
+
+header = ['Service-Name', 'Service-Status', 'App-Ready-Status', 'Down-Reason']
+header_detail = ['Service-Name', 'Service-Status', 'App-Ready-Status', 'Down-Reason', 'AppStatus-UpdateTime']
+
+SERVICE_STATUS_TABLE = 'ALL_SERVICE_STATUS'
+SYSREADY_TABLE = "SYSTEM_READY|SYSTEM_STATE"
+SERVICE_STATUS = 'service_status'
+APP_READY_STATUS = 'app_ready_status'
+FAIL_REASON = 'fail_reason'
+UPDATE_TIME = 'update_time'
+
+class SysreadyShow(object):
+    def __init__(self):
+        self.db = SonicV2Connector(host="127.0.0.1")
+        self.db.connect(self.db.STATE_DB)
+
+    def show(self, detailed_info):
+        keys = self.db.keys(self.db.STATE_DB, SERVICE_STATUS_TABLE + '*')
+        if not keys:
+            print('No system ready status data available - system-health service might be down\n')
+            return
+
+        sysready_state = self.db.get(self.db.STATE_DB, SYSREADY_TABLE, "Status")
+        if sysready_state == "UP":
+            print("System is ready\n")
+        else:
+            print("System is not ready - one or more services are not up\n")
+
+        #When brief option is specified, return here.
+        if detailed_info == False:
+            return
+
+        if detailed_info is None:
+            header_info = header
+        else:
+            header_info = header_detail
+
+        table = []
+        for key in natsorted(keys):
+            key_list = key.split('|')
+            if len(key_list) != 2: # error data in DB, log it and ignore
+                print('Warn: Invalid key in table {}: {}'.format(SERVICE_STATUS_TABLE, key))
+                continue
+
+            name = key_list[1]
+            data_dict = self.db.get_all(self.db.STATE_DB, key)
+            try:
+                service_status = data_dict[SERVICE_STATUS]
+                app_ready_status = data_dict[APP_READY_STATUS]
+                fail_reason = data_dict[FAIL_REASON]
+                update_time = data_dict[UPDATE_TIME]
+            except ValueError as e:
+                print('Error in data_dict')
+
+            if detailed_info is None:
+                table.append((name, service_status, app_ready_status, fail_reason))
+            else:
+                table.append((name, service_status, app_ready_status, fail_reason, update_time))
+
+
+        if table:
+            print(tabulate(table, header_info, tablefmt='simple', stralign='left'))
+        else:
+            print('No sysready status data available\n')
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Display the System Ready status',
+                                     formatter_class=argparse.RawTextHelpFormatter,
+                                     epilog="""
+                                      Examples:
+                                      sysreadyshow
+                                      sysreadyshow --brief
+                                      sysreadyshow --detail
+                                      """)
+
+    parser.add_argument('-b', '--brief', action='store_true', help='brief system ready status', default=False)
+    parser.add_argument('-d', '--detail', action='store_true', help='detailed system ready status', default=False)
+    args = parser.parse_args()
+
+    try:
+        sysready = SysreadyShow()
+        if args.detail:
+            detailed_info = True
+        elif args.brief:
+            detailed_info = False
+        else:
+            detailed_info = None
+        sysready.show(detailed_info)
+    except Exception as e:
+        print(str(e), file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

setup.py

@@ -150,7 +150,8 @@
         'scripts/memory_threshold_check_handler.py',
         'scripts/techsupport_cleanup.py',
         'scripts/storm_control.py',
-        'scripts/check_db_integrity.py'
+        'scripts/check_db_integrity.py',
+        'scripts/sysreadyshow'
     ],
     entry_points={
         'console_scripts': [

show/system_health.py

@@ -198,3 +198,34 @@ def monitor_list():
             entry.append(element[1]['type'])
             table.append(entry)
     click.echo(tabulate(table, header))
+
+
+@system_health.group('sysready-status',invoke_without_command=True)
+@click.pass_context
+def sysready_status(ctx):
+    """Show system-health system ready status"""
+
+    if ctx.invoked_subcommand is None:
+        try:
+            cmd = "sysreadyshow"
+            clicommon.run_command(cmd, display_cmd=False)
+        except Exception as e:
+            click.echo("Exception: {}".format(str(e)))
+
+
+@sysready_status.command('brief')
+def sysready_status_brief():
+    try:
+        cmd = "sysreadyshow --brief"
+        clicommon.run_command(cmd, display_cmd=False)
+    except Exception as e:
+        click.echo("Exception: {}".format(str(e)))
+
+
+@sysready_status.command('detail')
+def sysready_status_detail():
+    try:
+        cmd = "sysreadyshow --detail"
+        clicommon.run_command(cmd, display_cmd=False)
+    except Exception as e:
+        click.echo("Exception: {}".format(str(e)))

sonic_package_manager/manifest.py

@@ -177,6 +177,7 @@ def unmarshal(self, value):
             ManifestField('asic-service', DefaultMarshaller(bool), False),
             ManifestField('host-service', DefaultMarshaller(bool), True),
             ManifestField('delayed', DefaultMarshaller(bool), False),
+            ManifestField('check_up_status', DefaultMarshaller(bool), False),
             ManifestRoot('warm-shutdown', [
                 ManifestArray('after', DefaultMarshaller(str)),
                 ManifestArray('before', DefaultMarshaller(str)),

sonic_package_manager/service_creator/feature.py

@@ -202,4 +202,5 @@ def get_non_configurable_feature_entries(manifest) -> Dict[str, str]:
             'has_per_asic_scope': str(manifest['service']['asic-service']),
             'has_global_scope': str(manifest['service']['host-service']),
             'has_timer': str(manifest['service']['delayed']),
+            'check_up_status': str(manifest['service']['check_up_status']),
         }

tests/mock_tables/state_db.json

@@ -837,5 +837,32 @@
     },
     "LINK_TRAINING|Ethernet112": {
          "status": "off"
+    },
+    "ALL_SERVICE_STATUS|mgmt-framework": {
+      "app_ready_status": "OK",
+      "fail_reason": "-",
+      "service_status": "OK",
+      "update_time": "-"
+    },
+    "ALL_SERVICE_STATUS|swss": {
+      "app_ready_status": "OK",
+      "fail_reason": "-",
+      "service_status": "OK",
+      "update_time": "-"
+    },
+    "ALL_SERVICE_STATUS|bgp": {
+      "app_ready_status": "Down",
+      "fail_reason": "Inactive",
+      "service_status": "Down",
+      "update_time": "-"
+    },
+    "ALL_SERVICE_STATUS|pmon": {
+      "app_ready_status": "OK",
+      "fail_reason": "-",
+      "service_status": "OK",
+      "update_time": "-"
+    },
+    "SYSTEM_READY|SYSTEM_STATE": {
+        "Status":"DOWN"
     }
 }

tests/sonic_package_manager/test_service_creator.py

@@ -216,6 +216,7 @@ def test_feature_registration(mock_sonic_db, manifest):
         'has_per_asic_scope': 'False',
         'has_global_scope': 'True',
         'has_timer': 'False',
+        'check_up_status': 'False',
     })
 
 
@@ -228,6 +229,7 @@ def test_feature_update(mock_sonic_db, manifest):
         'has_per_asic_scope': 'False',
         'has_global_scope': 'True',
         'has_timer': 'False',
+        'check_up_status': 'False',
     }
     mock_connector = Mock()
     mock_connector.get_entry = Mock(return_value=curr_feature_config)
@@ -250,6 +252,7 @@ def test_feature_update(mock_sonic_db, manifest):
             'has_per_asic_scope': 'False',
             'has_global_scope': 'True',
             'has_timer': 'True',
+            'check_up_status': 'False',
         }),
     ], any_order=True)
 
@@ -270,6 +273,7 @@ def test_feature_registration_with_timer(mock_sonic_db, manifest):
         'has_per_asic_scope': 'False',
         'has_global_scope': 'True',
         'has_timer': 'True',
+        'check_up_status': 'False',
     })
 
 
@@ -288,6 +292,7 @@ def test_feature_registration_with_non_default_owner(mock_sonic_db, manifest):
         'has_per_asic_scope': 'False',
         'has_global_scope': 'True',
         'has_timer': 'False',
+        'check_up_status': 'False',
     })
 
 

tests/system_health_test.py

@@ -306,6 +306,43 @@ def test_health_detail(self):
 """
         assert result.output == expected
 
+    def test_health_systemready(self):
+        runner = CliRunner()
+        result = runner.invoke(show.cli.commands["system-health"].commands["sysready-status"])
+        click.echo(result.output)
+        print("myresult:{}".format(result.output))
+        expected = """\
+System is not ready - one or more services are not up
+
+Service-Name    Service-Status    App-Ready-Status    Down-Reason
+--------------  ----------------  ------------------  -------------
+bgp             Down              Down                Inactive
+mgmt-framework  OK                OK                  -
+pmon            OK                OK                  -
+swss            OK                OK                  -
+"""
+        assert result.output == expected
+        result = runner.invoke(show.cli.commands["system-health"].commands["sysready-status"],["brief"])
+        click.echo(result.output)
+        print("myresult:{}".format(result.output))
+        expected = """\
+System is not ready - one or more services are not up
+"""
+        assert result.output == expected
+        result = runner.invoke(show.cli.commands["system-health"].commands["sysready-status"],["detail"])
+        click.echo(result.output)
+        print("myresult:{}".format(result.output))
+        expected = """\
+System is not ready - one or more services are not up
+
+Service-Name    Service-Status    App-Ready-Status    Down-Reason    AppStatus-UpdateTime
+--------------  ----------------  ------------------  -------------  ----------------------
+bgp             Down              Down                Inactive       -
+mgmt-framework  OK                OK                  -              -
+pmon            OK                OK                  -              -
+swss            OK                OK                  -              -
+"""
+
     @classmethod
     def teardown_class(cls):
         print("TEARDOWN")