diff --git a/scripts/sysreadyshow b/scripts/sysreadyshow new file mode 100755 index 0000000000..2b95e75643 --- /dev/null +++ b/scripts/sysreadyshow @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 + +""" + Script to show system ready status. +""" + +import os +import sys +import argparse +from tabulate import tabulate +from natsort import natsorted + +# mock the redis for unit test purposes # +try: + if os.environ["UTILITIES_UNIT_TESTING"] == "1": + modules_path = os.path.join(os.path.dirname(__file__), "..") + test_path = os.path.join(modules_path, "tests") + sys.path.insert(0, modules_path) + sys.path.insert(0, test_path) + import mock_tables.dbconnector #lgtm [py/unused-import] +except KeyError: + pass + +from swsscommon.swsscommon import SonicV2Connector + +header = ['Service-Name', 'Service-Status', 'App-Ready-Status', 'Down-Reason'] +header_detail = ['Service-Name', 'Service-Status', 'App-Ready-Status', 'Down-Reason', 'AppStatus-UpdateTime'] + +SERVICE_STATUS_TABLE = 'ALL_SERVICE_STATUS' +SYSREADY_TABLE = "SYSTEM_READY|SYSTEM_STATE" +SERVICE_STATUS = 'service_status' +APP_READY_STATUS = 'app_ready_status' +FAIL_REASON = 'fail_reason' +UPDATE_TIME = 'update_time' + +class SysreadyShow(object): + def __init__(self): + self.db = SonicV2Connector(host="127.0.0.1") + self.db.connect(self.db.STATE_DB) + + def show(self, detailed_info): + keys = self.db.keys(self.db.STATE_DB, SERVICE_STATUS_TABLE + '*') + if not keys: + print('No system ready status data available - system-health service might be down\n') + return + + sysready_state = self.db.get(self.db.STATE_DB, SYSREADY_TABLE, "Status") + if sysready_state == "UP": + print("System is ready\n") + else: + print("System is not ready - one or more services are not up\n") + + #When brief option is specified, return here. + if detailed_info == False: + return + + if detailed_info is None: + header_info = header + else: + header_info = header_detail + + table = [] + for key in natsorted(keys): + key_list = key.split('|') + if len(key_list) != 2: # error data in DB, log it and ignore + print('Warn: Invalid key in table {}: {}'.format(SERVICE_STATUS_TABLE, key)) + continue + + name = key_list[1] + data_dict = self.db.get_all(self.db.STATE_DB, key) + try: + service_status = data_dict[SERVICE_STATUS] + app_ready_status = data_dict[APP_READY_STATUS] + fail_reason = data_dict[FAIL_REASON] + update_time = data_dict[UPDATE_TIME] + except ValueError as e: + print('Error in data_dict') + + if detailed_info is None: + table.append((name, service_status, app_ready_status, fail_reason)) + else: + table.append((name, service_status, app_ready_status, fail_reason, update_time)) + + + if table: + print(tabulate(table, header_info, tablefmt='simple', stralign='left')) + else: + print('No sysready status data available\n') + + +def main(): + parser = argparse.ArgumentParser(description='Display the System Ready status', + formatter_class=argparse.RawTextHelpFormatter, + epilog=""" + Examples: + sysreadyshow + sysreadyshow --brief + sysreadyshow --detail + """) + + parser.add_argument('-b', '--brief', action='store_true', help='brief system ready status', default=False) + parser.add_argument('-d', '--detail', action='store_true', help='detailed system ready status', default=False) + args = parser.parse_args() + + try: + sysready = SysreadyShow() + if args.detail: + detailed_info = True + elif args.brief: + detailed_info = False + else: + detailed_info = None + sysready.show(detailed_info) + except Exception as e: + print(str(e), file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index 9d0912fd1a..c954f03825 100644 --- a/setup.py +++ b/setup.py @@ -150,7 +150,8 @@ 'scripts/memory_threshold_check_handler.py', 'scripts/techsupport_cleanup.py', 'scripts/storm_control.py', - 'scripts/check_db_integrity.py' + 'scripts/check_db_integrity.py', + 'scripts/sysreadyshow' ], entry_points={ 'console_scripts': [ diff --git a/show/system_health.py b/show/system_health.py index 30d9d74114..845ac792cf 100644 --- a/show/system_health.py +++ b/show/system_health.py @@ -198,3 +198,34 @@ def monitor_list(): entry.append(element[1]['type']) table.append(entry) click.echo(tabulate(table, header)) + + +@system_health.group('sysready-status',invoke_without_command=True) +@click.pass_context +def sysready_status(ctx): + """Show system-health system ready status""" + + if ctx.invoked_subcommand is None: + try: + cmd = "sysreadyshow" + clicommon.run_command(cmd, display_cmd=False) + except Exception as e: + click.echo("Exception: {}".format(str(e))) + + +@sysready_status.command('brief') +def sysready_status_brief(): + try: + cmd = "sysreadyshow --brief" + clicommon.run_command(cmd, display_cmd=False) + except Exception as e: + click.echo("Exception: {}".format(str(e))) + + +@sysready_status.command('detail') +def sysready_status_detail(): + try: + cmd = "sysreadyshow --detail" + clicommon.run_command(cmd, display_cmd=False) + except Exception as e: + click.echo("Exception: {}".format(str(e))) diff --git a/sonic_package_manager/manifest.py b/sonic_package_manager/manifest.py index 2d9f3514e7..94e00dec32 100644 --- a/sonic_package_manager/manifest.py +++ b/sonic_package_manager/manifest.py @@ -177,6 +177,7 @@ def unmarshal(self, value): ManifestField('asic-service', DefaultMarshaller(bool), False), ManifestField('host-service', DefaultMarshaller(bool), True), ManifestField('delayed', DefaultMarshaller(bool), False), + ManifestField('check_up_status', DefaultMarshaller(bool), False), ManifestRoot('warm-shutdown', [ ManifestArray('after', DefaultMarshaller(str)), ManifestArray('before', DefaultMarshaller(str)), diff --git a/sonic_package_manager/service_creator/feature.py b/sonic_package_manager/service_creator/feature.py index 73e35566dd..f9ec606532 100644 --- a/sonic_package_manager/service_creator/feature.py +++ b/sonic_package_manager/service_creator/feature.py @@ -202,4 +202,5 @@ def get_non_configurable_feature_entries(manifest) -> Dict[str, str]: 'has_per_asic_scope': str(manifest['service']['asic-service']), 'has_global_scope': str(manifest['service']['host-service']), 'has_timer': str(manifest['service']['delayed']), + 'check_up_status': str(manifest['service']['check_up_status']), } diff --git a/tests/mock_tables/state_db.json b/tests/mock_tables/state_db.json index fc17df6e83..8a323717af 100644 --- a/tests/mock_tables/state_db.json +++ b/tests/mock_tables/state_db.json @@ -837,5 +837,32 @@ }, "LINK_TRAINING|Ethernet112": { "status": "off" + }, + "ALL_SERVICE_STATUS|mgmt-framework": { + "app_ready_status": "OK", + "fail_reason": "-", + "service_status": "OK", + "update_time": "-" + }, + "ALL_SERVICE_STATUS|swss": { + "app_ready_status": "OK", + "fail_reason": "-", + "service_status": "OK", + "update_time": "-" + }, + "ALL_SERVICE_STATUS|bgp": { + "app_ready_status": "Down", + "fail_reason": "Inactive", + "service_status": "Down", + "update_time": "-" + }, + "ALL_SERVICE_STATUS|pmon": { + "app_ready_status": "OK", + "fail_reason": "-", + "service_status": "OK", + "update_time": "-" + }, + "SYSTEM_READY|SYSTEM_STATE": { + "Status":"DOWN" } } diff --git a/tests/sonic_package_manager/test_service_creator.py b/tests/sonic_package_manager/test_service_creator.py index c943289362..f82e2e519f 100644 --- a/tests/sonic_package_manager/test_service_creator.py +++ b/tests/sonic_package_manager/test_service_creator.py @@ -216,6 +216,7 @@ def test_feature_registration(mock_sonic_db, manifest): 'has_per_asic_scope': 'False', 'has_global_scope': 'True', 'has_timer': 'False', + 'check_up_status': 'False', }) @@ -228,6 +229,7 @@ def test_feature_update(mock_sonic_db, manifest): 'has_per_asic_scope': 'False', 'has_global_scope': 'True', 'has_timer': 'False', + 'check_up_status': 'False', } mock_connector = Mock() mock_connector.get_entry = Mock(return_value=curr_feature_config) @@ -250,6 +252,7 @@ def test_feature_update(mock_sonic_db, manifest): 'has_per_asic_scope': 'False', 'has_global_scope': 'True', 'has_timer': 'True', + 'check_up_status': 'False', }), ], any_order=True) @@ -270,6 +273,7 @@ def test_feature_registration_with_timer(mock_sonic_db, manifest): 'has_per_asic_scope': 'False', 'has_global_scope': 'True', 'has_timer': 'True', + 'check_up_status': 'False', }) @@ -288,6 +292,7 @@ def test_feature_registration_with_non_default_owner(mock_sonic_db, manifest): 'has_per_asic_scope': 'False', 'has_global_scope': 'True', 'has_timer': 'False', + 'check_up_status': 'False', }) diff --git a/tests/system_health_test.py b/tests/system_health_test.py index 819b986ca5..2f9e48e722 100644 --- a/tests/system_health_test.py +++ b/tests/system_health_test.py @@ -306,6 +306,43 @@ def test_health_detail(self): """ assert result.output == expected + def test_health_systemready(self): + runner = CliRunner() + result = runner.invoke(show.cli.commands["system-health"].commands["sysready-status"]) + click.echo(result.output) + print("myresult:{}".format(result.output)) + expected = """\ +System is not ready - one or more services are not up + +Service-Name Service-Status App-Ready-Status Down-Reason +-------------- ---------------- ------------------ ------------- +bgp Down Down Inactive +mgmt-framework OK OK - +pmon OK OK - +swss OK OK - +""" + assert result.output == expected + result = runner.invoke(show.cli.commands["system-health"].commands["sysready-status"],["brief"]) + click.echo(result.output) + print("myresult:{}".format(result.output)) + expected = """\ +System is not ready - one or more services are not up +""" + assert result.output == expected + result = runner.invoke(show.cli.commands["system-health"].commands["sysready-status"],["detail"]) + click.echo(result.output) + print("myresult:{}".format(result.output)) + expected = """\ +System is not ready - one or more services are not up + +Service-Name Service-Status App-Ready-Status Down-Reason AppStatus-UpdateTime +-------------- ---------------- ------------------ ------------- ---------------------- +bgp Down Down Inactive - +mgmt-framework OK OK - - +pmon OK OK - - +swss OK OK - - +""" + @classmethod def teardown_class(cls): print("TEARDOWN")