Add diagnostic_utils with unit tests (#160)

Add neon-upload-diagnostics entrypoint
Depreciate neon_core_client module
Update dependencies

neon_core/run_neon.py

@@ -162,7 +162,7 @@ def start_neon():
         _start_process("neon_core_server")
     else:
         _start_process("neon_enclosure_client")
-        _start_process("neon_core_client")
+        # _start_process("neon_core_client")
         _start_process("mycroft-gui-app")
         _start_process(["python3", "-m", "neon_core.gui"])
 

neon_core/skills/service.py

@@ -35,9 +35,12 @@
 from neon_core.skills.intent_service import NeonIntentService
 from neon_core.skills.skill_manager import NeonSkillManager
 from neon_utils.configuration_utils import get_neon_skills_config, \
-    get_neon_lang_config
+    get_neon_lang_config, get_neon_local_config
 from neon_utils.net_utils import check_online
 
+from neon_core.util.diagnostic_utils import report_metric
+from neon_utils.metrics_utils import announce_connection
+
 
 def on_started():
     LOG.info('Skills service is starting up.')
@@ -99,6 +102,20 @@ def start(self):
         while not self.skill_manager.is_all_loaded():
             time.sleep(0.1)
         self.status.set_ready()
+        announce_connection()
+
+    def _initialize_metrics_handler(self):
+        """
+        Start bus listener for metrics
+        """
+        def handle_metric(message):
+            report_metric(message.data.pop("name"), **message.data)
+
+        if get_neon_local_config()['prefFlags']['metrics']:
+            LOG.info("Metrics reporting enabled")
+            self.bus.on("neon.metric", handle_metric)
+        else:
+            LOG.info("Metrics reporting disabled")
 
     def _register_intent_services(self):
         """Start up the all intent services and connect them as needed.

neon_core/util/diagnostic_utils.py

@@ -0,0 +1,116 @@
+# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System
+#
+# Copyright 2008-2021 Neongecko.com Inc. | All Rights Reserved
+#
+# Notice of License - Duplicating this Notice of License near the start of any file containing
+# a derivative of this software is a condition of license for this software.
+# Friendly Licensing:
+# No charge, open source royalty free use of the Neon AI software source and object is offered for
+# educational users, noncommercial enthusiasts, Public Benefit Corporations (and LLCs) and
+# Social Purpose Corporations (and LLCs). Developers can contact developers@neon.ai
+# For commercial licensing, distribution of derivative works or redistribution please contact licenses@neon.ai
+# Distributed on an "AS IS” basis without warranties or conditions of any kind, either express or implied.
+# Trademarks of Neongecko: Neon AI(TM), Neon Assist (TM), Neon Communicator(TM), Klat(TM)
+# Authors: Guy Daniels, Daniel McKnight, Regina Bloomstine, Elon Gasper, Richard Leeds
+#
+# Specialized conversational reconveyance options from Conversation Processing Intelligence Corp.
+# US Patents 2008-2021: US7424516, US20140161250, US20140177813, US8638908, US8068604, US8553852, US10530923, US10530924
+# China Patent: CN102017585  -  Europe Patent: EU2156652  -  Patents Pending
+import json
+import socket
+import glob
+import os
+
+from neon_utils import LOG
+from neon_utils.metrics_utils import report_metric
+from neon_utils.configuration_utils import NGIConfig
+
+
+def send_diagnostics(allow_logs=True, allow_transcripts=True, allow_config=True):
+    """
+    Uploads diagnostics to the configured server. Default data includes start.log and basic system information.
+    If logs are allowed, current core logs will be uploaded.
+    If transcripts are allowed, recent transcripts will be uploaded.
+    If config is allowed, local and user configuration files will be uploaded.
+    :param allow_logs: Allows uploading current log files
+    :param allow_transcripts: Allows uploading recent transcriptions
+    :param allow_config: Allows uploading Neon config files
+    """
+    LOG.info(f"Sending Diagnostics: logs={allow_logs} transcripts={allow_transcripts} config={allow_config}")
+    # Get Configurations
+    local_configuration = NGIConfig("ngi_local_conf").content
+    user_configuration = NGIConfig("ngi_user_info").content
+    # auth_configuration = NGIConfig("ngi_auth_vars").content
+    if allow_config:
+        configs = {"local": local_configuration,
+                   "user": user_configuration}
+    else:
+        configs = None
+
+    # Get Logs
+    logs_dir = os.path.expanduser(local_configuration["dirVars"]["logsDir"])
+    startup_log = os.path.join(logs_dir, "start.log")
+    if os.path.isfile(startup_log):
+        with open(startup_log, 'r') as start:
+            startup = start.read()
+            # Catch a very large log and take last 100000 chars, rounded to a full line
+            if len(startup) > 100000:
+                startup = startup[-100000:].split("\n", 1)[1]
+    else:
+        startup = None
+    if allow_logs:
+        logs = dict()
+        try:
+            for log in glob.glob(f'{logs_dir}/*.log'):
+                if os.path.basename(log) == "start.log":
+                    pass
+                with open(log, 'r') as f:
+                    contents = f.read()
+                    # Catch a very large log and take last 100000 chars, rounded to a full line
+                    if len(contents) > 100000:
+                        contents = contents[-100000:].split("\n", 1)[1]
+                    logs[os.path.basename(os.path.splitext(log)[0])] = contents
+            # TODO: + last few archived logs, testing logs DM
+        except Exception as e:
+            LOG.error(e)
+    else:
+        logs = None
+
+    transcript_file = os.path.join(os.path.expanduser(local_configuration["dirVars"]["docsDir"]),
+                                   "csv_files", "full_ts.csv")
+    if allow_transcripts and os.path.isfile(transcript_file):
+        with open(transcript_file, "r") as f:
+            lines = f.readlines()
+            try:
+                transcripts = lines[-500:]
+            except Exception as e:
+                LOG.error(e)
+                transcripts = lines
+            transcripts = "".join(transcripts)
+    else:
+        transcripts = None
+
+    data = {"host": socket.gethostname(),
+            "startup": startup,
+            "configurations": json.dumps(configs),
+            "logs": json.dumps(logs),
+            "transcripts": transcripts}
+    report_metric("diagnostics", **data)
+    return data
+
+
+def cli_send_diags():
+    """
+    CLI Entry Point to Send Diagnostics
+    """
+    import argparse
+    parser = argparse.ArgumentParser(description="Upload Neon Diagnostics Files", add_help=True)
+    parser.add_argument("--no-transcripts", dest="transcripts", default=True, action='store_false',
+                        help="Disable upload of all transcribed input")
+    parser.add_argument("--no-logs", dest="logs", default=True, action='store_false',
+                        help="Disable upload of Neon log files (NOTE: start_neon.log is always uploaded)")
+    parser.add_argument("--no-config", dest="config", default=True, action='store_false',
+                        help="Disable upload of Neon config files")
+
+    args = parser.parse_args()
+    send_diagnostics(args.logs, args.transcripts, args.config)

requirements/client.txt

@@ -1,4 +1,3 @@
-neon-client @ git+https://github.com/NeonGeckoCom/neon-core-client
 neon-transcripts-controller @ git+https://github.com/NeonGeckoCom/transcripts_controller
 
 # wake word plugins

requirements/requirements.txt

@@ -8,7 +8,7 @@ neon_audio>=0.3.2
 neon_enclosure>=0.1.0
 
 # utils
-neon-utils>=0.10.2
+neon-utils>=0.12.0
 rapidfuzz
 kthread
 ovos_utils>=0.0.12a3

setup.py

@@ -91,6 +91,7 @@ def get_requirements(requirements_filename: str):
             'neon_skills_service=neon_core.skills.__main__:main',
             'neon_gui_service=neon_core.gui.__main__:main',
             'neon-install-default-skills=neon_core.util.skill_utils:install_skills_default',
+            'neon-upload-diagnostics=neon_core.util.diagnostic_utils:cli_send_diags',
             'neon-start=neon_core.run_neon:start_neon',
             'neon-stop=neon_core.run_neon:stop_neon'
         ]

test/diagnostic_files/audio.log

@@ -0,0 +1,23 @@
+2021-10-26 14:05:55.196 - neon-utils - neon_utils.configuration_utils:_move_config_sections:779 - WARNING - Depreciated keys found in user config! Adding them to local config
+2021-10-26 14:05:55.451 - neon-utils - neon_utils.configuration_utils:_write_yaml_file:309 - DEBUG - YAML updated ngi_local_conf
+2021-10-26 14:05:55.503 - neon-utils - neon_utils.configuration_utils:dict_make_equal_keys:525 - WARNING - Removing 'phonemes' from dict!
+2021-10-26 14:05:55.768 - neon-utils - neon_utils.configuration_utils:_write_yaml_file:309 - DEBUG - YAML updated ngi_local_conf
+2021-10-26 14:05:55.872 - neon-utils - neon_utils.configuration_utils:get_neon_local_config:856 - INFO - Loaded local config from /home/d_mcknight/PycharmProjects/_.Core/NeonCore/ngi_local_conf.yml
+2021-10-26 14:05:59.996 | INFO     | 375209 | mycroft.messagebus.load_config:load_message_bus_config:33 | Loading message bus configs
+2021-10-26 14:05:59.998 | ERROR    | 375209 | mycroft_bus_client.client.client | === ConnectionRefusedError(111, 'Connection refused') ===
+Traceback (most recent call last):
+  File "/home/d_mcknight/PycharmProjects/_.Core/venv/lib/python3.8/site-packages/websocket/_app.py", line 312, in run_forever
+    self.sock.connect(
+  File "/home/d_mcknight/PycharmProjects/_.Core/venv/lib/python3.8/site-packages/websocket/_core.py", line 249, in connect
+    self.sock, addrs = connect(url, self.sock_opt, proxy_info(**options),
+  File "/home/d_mcknight/PycharmProjects/_.Core/venv/lib/python3.8/site-packages/websocket/_http.py", line 130, in connect
+    sock = _open_socket(addrinfo_list, options.sockopt, options.timeout)
+  File "/home/d_mcknight/PycharmProjects/_.Core/venv/lib/python3.8/site-packages/websocket/_http.py", line 208, in _open_socket
+    raise err
+  File "/home/d_mcknight/PycharmProjects/_.Core/venv/lib/python3.8/site-packages/websocket/_http.py", line 185, in _open_socket
+    sock.connect(address)
+ConnectionRefusedError: [Errno 111] Connection refused
+2021-10-26 14:05:59.999 | ERROR    | 375209 | mycroft_bus_client.client.client | Exception closing websocket: ConnectionRefusedError(111, 'Connection refused')
+2021-10-26 14:05:59.999 | WARNING  | 375209 | mycroft_bus_client.client.client | Message Bus Client will reconnect in 5.0 seconds.
+2021-10-26 14:06:05.005 | INFO     | 375209 | mycroft_bus_client.client.client | Connected
+2021-10-26 14:06:05.006 | INFO     | 375209 | mycroft.util.process_utils:start_message_bus_client:155 | Connected to messagebus

test/diagnostic_files/bus.log

@@ -0,0 +1,3 @@
+2021-10-26 14:05:54.270 - OVOS - ovos_utils.system:set_root_path:60 - INFO - mycroft root set to NeonCore
+2021-10-26 14:05:54.288 - OVOS - ovos_utils.configuration:set_xdg_base:33 - INFO - XDG base folder set to: 'neon_core'
+2021-10-26 14:05:54.314 - OVOS - ovos_utils.configuration:set_config_filename:45 - INFO - config filename set to: 'neon'

test/diagnostic_files/csv_files/full_ts.csv

@@ -0,0 +1,3 @@
+Date,Time,Profile,Client,Input,Location,Wav_Length
+2021-08-06,13:45:08,,local,hello,,
+2021-08-06,14:07:14,,local,what time is it,,

test/diagnostic_files/display.log

@@ -0,0 +1,3 @@
+2021-10-26 14:05:54.857 - OVOS - ovos_utils.system:set_root_path:60 - INFO - mycroft root set to NeonCore
+2021-10-26 14:05:54.885 - OVOS - ovos_utils.configuration:set_xdg_base:33 - INFO - XDG base folder set to: 'neon_core'
+2021-10-26 14:05:54.913 - OVOS - ovos_utils.configuration:set_config_filename:45 - INFO - config filename set to: 'neon'

test/diagnostic_files/enclosure.log

@@ -0,0 +1 @@
+2021-10-26 15:13:29.713 - neon-utils - neon_enclosure.client.enclosure.__main__:on_stopping:36 - INFO - Enclosure is shutting down...

test/diagnostic_files/gui.log

@@ -0,0 +1,15 @@
+qt5ct: using qt5ct plugin
+Attribute Qt::AA_ShareOpenGLContexts must be set before QCoreApplication is created.
+qrc:/main.qml:38:5: Unable to assign [undefined] to int
+qrc:/main.qml:37:5: Unable to assign [undefined] to int
+qrc:/main.qml:36:5: Unable to assign [undefined] to int
+qrc:/main.qml:35:5: Unable to assign [undefined] to int
+qrc:/main.qml:34:5: Unable to assign [undefined] to int
+qrc:/main.qml:33:5: Unable to assign [undefined] to int
+qrc:/main.qml:225:17: Unable to assign ApplicationWindow_QMLTYPE_68 to QQuickItem
+qrc:/main.qml:252:17: Unable to assign null to QStringList
+qml: Trying to connect to Mycroft
+/usr/bin/mycroft-gui-core-loader: 8: cd: can't cd to /home/d_mcknight/mycroft-core
+Main Socket connected, trying to connect gui
+Received port 18181 for gui "{23153746-1acd-4ba1-9630-1edc0d0b5ab9}"
+

test/diagnostic_files/skills.log

@@ -0,0 +1,3 @@
+2021-10-26 14:05:54.517 - OVOS - ovos_utils.system:set_root_path:60 - INFO - mycroft root set to NeonCore
+2021-10-26 14:05:54.528 - OVOS - ovos_utils.configuration:set_xdg_base:33 - INFO - XDG base folder set to: 'neon_core'
+2021-10-26 14:05:54.568 - OVOS - ovos_utils.configuration:set_config_filename:45 - INFO - config filename set to: 'neon'

test/diagnostic_files/start.log

@@ -0,0 +1 @@
+start log

test/diagnostic_files/voice.log

@@ -0,0 +1 @@
+voice_log

test/test_diagnostic_utils.py

@@ -0,0 +1,132 @@
+# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System
+# # All trademark and other rights reserved by their respective owners
+# # Copyright 2008-2021 Neongecko.com Inc.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 1. Redistributions of source code must retain the above copyright notice,
+#    this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+# 3. Neither the name of the copyright holder nor the names of its
+#    contributors may be used to endorse or promote products derived from this
+#    software without specific prior written permission.
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS  BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+# OR PROFITS;  OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE,  EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import os
+import shutil
+import sys
+import unittest
+import neon_utils.metrics_utils
+
+from mock import Mock
+
+from neon_utils import get_neon_local_config
+
+sys.path.append(os.path.dirname(os.path.dirname(__file__)))
+
+
+class DiagnosticUtilsTests(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        cls.report_metric = Mock()
+        cls.config_dir = os.path.join(os.path.dirname(__file__), "test_config")
+        os.makedirs(cls.config_dir)
+
+        os.environ["NEON_CONFIG_PATH"] = cls.config_dir
+        test_dir = os.path.join(os.path.dirname(__file__), "diagnostic_files")
+        local_config = get_neon_local_config()
+        local_config["dirVars"]["diagsDir"] = test_dir
+        local_config["dirVars"]["docsDir"] = test_dir
+        local_config["dirVars"]["logsDir"] = test_dir
+        local_config["dirVars"]["diagsDir"] = test_dir
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        if os.getenv("NEON_CONFIG_PATH"):
+            os.environ.pop("NEON_CONFIG_PATH")
+        shutil.rmtree(cls.config_dir)
+
+    def setUp(self) -> None:
+        self.report_metric.reset_mock()
+        neon_utils.metrics_utils.report_metric = self.report_metric
+
+
+    def test_send_diagnostics_default(self):
+        from neon_core.util.diagnostic_utils import send_diagnostics
+        send_diagnostics()
+        self.report_metric.assert_called_once()
+        args = self.report_metric.call_args
+        self.assertEqual(args.args, ("diagnostics",))
+        data = args.kwargs
+        self.assertIsInstance(data, dict)
+        self.assertIsInstance(data["host"], str)
+        self.assertIsInstance(data["configurations"], dict)
+        self.assertIsInstance(data["logs"], dict)
+        self.assertIsInstance(data["transcripts"], str)
+
+    def test_send_diagnostics_no_extras(self):
+        from neon_core.util.diagnostic_utils import send_diagnostics
+        send_diagnostics(False, False, False)
+        self.report_metric.assert_called_once()
+        args = self.report_metric.call_args
+        self.assertEqual(args.args, ("diagnostics",))
+        data = args.kwargs
+        self.assertIsInstance(data, dict)
+        self.assertIsInstance(data["host"], str)
+        self.assertIsNone(data["configurations"])
+        self.assertIsNone(data["logs"])
+        self.assertIsNone(data["transcripts"])
+
+    def test_send_diagnostics_allow_logs(self):
+        from neon_core.util.diagnostic_utils import send_diagnostics
+        send_diagnostics(True, False, False)
+        self.report_metric.assert_called_once()
+        args = self.report_metric.call_args
+        self.assertEqual(args.args, ("diagnostics",))
+        data = args.kwargs
+        self.assertIsInstance(data, dict)
+        self.assertIsInstance(data["host"], str)
+        self.assertIsNone(data["configurations"])
+        self.assertIsInstance(data["logs"], dict)
+        self.assertIsNone(data["transcripts"])
+
+    def test_send_diagnostics_allow_transcripts(self):
+        from neon_core.util.diagnostic_utils import send_diagnostics
+        send_diagnostics(False, True, False)
+        self.report_metric.assert_called_once()
+        args = self.report_metric.call_args
+        self.assertEqual(args.args, ("diagnostics",))
+        data = args.kwargs
+        self.assertIsInstance(data, dict)
+        self.assertIsInstance(data["host"], str)
+        self.assertIsNone(data["configurations"])
+        self.assertIsNone(data["logs"])
+        self.assertIsInstance(data["transcripts"], str)
+
+    def test_send_diagnostics_allow_config(self):
+        from neon_core.util.diagnostic_utils import send_diagnostics
+        send_diagnostics(False, False, True)
+        self.report_metric.assert_called_once()
+        args = self.report_metric.call_args
+        self.assertEqual(args.args, ("diagnostics",))
+        data = args.kwargs
+        self.assertIsInstance(data, dict)
+        self.assertIsInstance(data["host"], str)
+        self.assertIsInstance(data["configurations"], dict)
+        self.assertIsNone(data["logs"])
+        self.assertIsNone(data["transcripts"])
+
+
+if __name__ == '__main__':
+    unittest.main()

test/test_run_neon.py

@@ -124,11 +124,11 @@ def test_audio_module(self):
     #     matches = resp.data.get("transcripts")
     #     self.assertIsInstance(matches, list)
 
-    def test_client_module(self):
-        resp = self.bus.wait_for_response(Message("neon.client.update_brands"), "neon.server.update_brands.response")
-        self.assertIsInstance(resp, Message)
-        data = resp.data
-        self.assertIsInstance(data["success"], bool)
+    # def test_client_module(self):
+    #     resp = self.bus.wait_for_response(Message("neon.client.update_brands"), "neon.server.update_brands.response")
+    #     self.assertIsInstance(resp, Message)
+    #     data = resp.data
+    #     self.assertIsInstance(data["success"], bool)
 
     def test_skills_module(self):
         response = self.bus.wait_for_response(Message('mycroft.skills.is_ready'))