Specify default skills as python dependencies

Dockerfile

@@ -31,4 +31,6 @@ RUN chmod ugo+x /root/run.sh
 CMD ["/root/run.sh"]
 
 FROM base as default_skills
-RUN neon-install-default-skills
+RUN pip install .[skills_required,skills_essential,skills_default,skills_extended]
+# TODO: Default skill installation is a temporary step until all skills are pip installable
+RUN neon-install-default-skills

docker/config/neon.yaml

@@ -2,6 +2,8 @@ websocket:
   host: neon-messagebus
 gui_websocket:
   host: neon-gui
+gui:
+  idle_display_skill: skill-homescreen-lite.openvoiceos
 ready_settings:
   - skills
   - speech

docker_overlay/etc/neon/neon.yaml

@@ -1,7 +1,6 @@
 skills:
   run_gui_file_server: true
   wait_for_internet: false
-  directory: /default_skills
   extra_directories:
   - /default_skills
   - /skills
@@ -34,7 +33,9 @@ skills:
   install_essential: true
   essential_skills: []
   install_default: false
-  default_skills: https://raw.githubusercontent.com/NeonGeckoCom/neon_skills/master/skill_lists/DEFAULT-SKILLS-DOCKER
+  default_skills:
+  # TODO: Defaults are just patching skills not yet pip installable
+  - https://github.com/JarbasSkills/skill-icanhazdadjokes/tree/dev
 play_wav_cmdline: "play %1"
 play_mp3_cmdline: "play %1"
 play_ogg_cmdline: "play %1"

docker_overlay/root/.config/neon/skills/skill-instructions.neongeckocom/settings.json

@@ -0,0 +1,4 @@
+{
+  "prompt_on_start": false,
+  "__mycroft_skill_firstrun": false
+}

neon_core/configuration/__init__.py

@@ -38,7 +38,7 @@
 
 
 def get_private_keys():
-    return Configuration.get(remote=False).get("keys", {})
+    return Configuration().get("keys", {})
 
 
 def patch_config(config: dict = None):

neon_core/skills/service.py

@@ -28,11 +28,16 @@
 
 import time
 
+from tempfile import gettempdir
+from os import listdir
+from os.path import isdir, dirname, join
 from typing import Optional
 from threading import Thread
 
+from ovos_config.locale import set_default_lang, set_default_tz
 from ovos_config.config import Configuration
 from ovos_utils.log import LOG
+from ovos_utils.skills.locations import get_plugin_skills, get_skill_directories
 from neon_utils.metrics_utils import announce_connection
 from neon_utils.signal_utils import init_signal_handlers, init_signal_bus
 from neon_utils.messagebus_utils import get_messagebus
@@ -45,7 +50,6 @@
 
 from mycroft.skills.api import SkillApi
 from mycroft.skills.event_scheduler import EventScheduler
-from ovos_config.locale import set_default_lang, set_default_tz
 from mycroft.util.process_utils import ProcessStatus, StatusCallbackMap
 
 
@@ -91,14 +95,45 @@ def __init__(self,
                                            on_ready=ready_hook,
                                            on_error=error_hook,
                                            on_stopping=stopping_hook)
-        self.config = Configuration()
-
+        # Apply any passed config values and load Configuration
         if config:
             LOG.info("Updating global config with passed config")
             from neon_core.configuration import patch_config
             patch_config(config)
-            assert all((self.config["skills"][x] == config["skills"][x]
-                        for x in config["skills"]))
+        self.config = Configuration()
+
+    def _init_gui_server(self):
+        """
+        If configured, start the local file server to serve QML resources
+        """
+        from os.path import basename, isdir, join
+        from os import symlink, makedirs
+        # from shutil import copytree
+        if not self.config["skills"].get("run_gui_file_server"):
+            return
+        directory = join(gettempdir(), "neon", "qml", "skills")
+        if not isdir(directory):
+            makedirs(directory, exist_ok=True)
+        for d in reversed(self._get_skill_dirs()):
+            if not isdir(join(d, "ui")):
+                continue
+            skill_dir = basename(d)
+            if not isdir(join(directory, skill_dir)):
+                makedirs(join(directory, skill_dir))
+                symlink(join(d, "ui"), join(directory, skill_dir, "ui"))
+                LOG.info(f"linked {d}/ui to {directory}/{skill_dir}/ui")
+        self.http_server = start_qml_http_server(directory)
+
+    def _get_skill_dirs(self) -> list:
+        """
+        Get a list of paths to every loaded skill in load order (priority last)
+        """
+        plugin_dirs, _ = get_plugin_skills()
+        skill_base_dirs = get_skill_directories(self.config)
+
+        skill_dirs = [join(base_dir, d) for base_dir in skill_base_dirs
+                      for d in listdir(base_dir)]
+        return plugin_dirs + skill_dirs
 
     def run(self):
         # Set the active lang to match the configured one
@@ -107,25 +142,35 @@ def run(self):
         # Set the default timezone to match the configured one
         set_default_tz()
 
+        # Setup signal manager
         self.bus = self.bus or get_messagebus()
         init_signal_bus(self.bus)
         init_signal_handlers()
+
+        # Setup Intents and Skill Manager
         self._register_intent_services()
         self.event_scheduler = EventScheduler(self.bus)
         self.status = ProcessStatus('skills', self.bus, self.callbacks,
                                     namespace="neon")
         SkillApi.connect_bus(self.bus)
+        LOG.info("Starting Skill Manager")
         self.skill_manager = NeonSkillManager(self.bus, self.watchdog)
         self.skill_manager.setName("skill_manager")
         self.skill_manager.start()
+        LOG.info("Skill Manager started")
 
-        skill_dir = self.skill_manager.get_default_skills_dir()
-        if self.config["skills"].get("run_gui_file_server"):
-            self.http_server = start_qml_http_server(skill_dir)
+        # Setup GUI File Server
+        try:
+            self._init_gui_server()
+        except Exception as e:
+            # Allow service to start if GUI file server fails
+            LOG.exception(e)
 
+        # Update status
         self.status.set_started()
 
         # TODO: These should be event-based in Mycroft/OVOS
+        # Wait for skill manager to start up
         while not self.skill_manager.is_alive():
             time.sleep(0.1)
         self.status.set_alive()

neon_core/util/qml_file_server.py

@@ -33,6 +33,7 @@
 from tempfile import gettempdir
 from os.path import isdir, join, dirname
 from threading import Thread, Event
+from neon_utils.logger import LOG
 
 _HTTP_SERVER: socketserver.TCPServer = None
 
@@ -54,16 +55,21 @@ def start_qml_http_server(skills_dir: str, port: int = 8000):
     system_dir = join(dirname(dirname(__file__)), "res")
 
     qml_dir = join(gettempdir(), "neon", "qml")
-    os.makedirs(qml_dir, exist_ok=True)
+    if not isdir(qml_dir):
+        os.makedirs(qml_dir)
 
     served_skills_dir = join(qml_dir, "skills")
     served_system_dir = join(qml_dir, "system")
-    if os.path.exists(served_skills_dir) or os.path.islink(served_skills_dir):
-        os.remove(served_skills_dir)
-    if os.path.exists(served_system_dir) or os.path.islink(served_skills_dir):
-        os.remove(served_system_dir)
 
-    os.symlink(skills_dir, served_skills_dir)
+    # If serving from a temporary linked directory, create a fresh symlink
+    if skills_dir != served_skills_dir:
+        LOG.info(f"Linking {skills_dir} to {served_skills_dir}")
+        if os.path.exists(served_skills_dir) or os.path.islink(served_skills_dir):
+            os.remove(served_skills_dir)
+        os.symlink(skills_dir, served_skills_dir)
+
+    if os.path.exists(served_system_dir) or os.path.islink(served_system_dir):
+        os.remove(served_system_dir)
     os.symlink(system_dir, served_system_dir)
     started_event = Event()
     http_daemon = Thread(target=_initialize_http_server,

requirements/requirements.txt

@@ -2,8 +2,8 @@
 ovos-core[skills_lgpl]~=0.0.3,>=0.0.4a38
 # utils
 neon-utils[network,configuration]>=1.0.0a25,<2.0.0
-ovos_utils~=0.0,>=0.0.23a6
-ovos-config~=0.0,>=0.0.4a4
+ovos_utils~=0.0,>=0.0.23
+ovos-config~=0.0,>=0.0.4
 ovos-skills-manager~=0.0.10,>=0.0.11a5
 ovos-plugin-manager~=0.0.16
 

requirements/skills_default.txt

@@ -0,0 +1,16 @@
+ovos-skill-homescreen-lite @ git+https://github.com/openvoiceos/skill-homescreen-lite@TEST_SimpleHomescreen
+skill-ddg @ git+https://github.com/openvoiceos/skill-ddg
+neon-skill-alerts
+neon-skill-caffeinewiz
+neon-skill-data_controls
+neon-skill-fallback_wolfram_alpha
+neon-skill-personal
+neon-skill-speak
+neon-skill-speed_test
+neon-skill-spelling
+neon-skill-stock
+neon-skill-support_helper
+# neon-skill-update
+neon-skill-user_settings
+neon-skill-weather
+neon-skill-wikipedia

requirements/skills_essential.txt

@@ -0,0 +1,5 @@
+neon-skill-about
+neon-skill-date_time
+# neon-skill-demo
+neon-skill-device_controls
+neon-skill-ip_address

requirements/skills_extended.txt

@@ -0,0 +1,11 @@
+neon-skill-audio_record
+neon-skill-custom_conversation
+neon-skill-instructions
+neon-skill-launcher
+neon-skill-messaging
+neon-skill-news
+# neon-skill-recipes
+neon-skill-synonyms
+neon-skill-translation
+neon-skill-avmusic
+neon-skill-camera

requirements/skills_required.txt

@@ -0,0 +1,3 @@
+ovos-skill-stop @ git+https://github.com/openvoiceos/skill-ovos-stop
+neon-skill-fallback_unknown
+neon-skill-communication

setup.py

@@ -83,7 +83,11 @@ def get_requirements(requirements_filename: str):
         "vision": get_requirements("vision.txt"),
         "test": get_requirements("test.txt"),
         "pi": get_requirements("pi.txt"),
-        "docker": get_requirements("docker.txt")
+        "docker": get_requirements("docker.txt"),
+        "skills_required": get_requirements("skills_required.txt"),
+        "skills_essential": get_requirements("skills_essential.txt"),
+        "skills_default": get_requirements("skills_default.txt"),
+        "skills_extended": get_requirements("skills_extended.txt")
     },
     packages=find_packages(include=['neon_core*']),
     package_data={'neon_core': ['res/precise_models/*', 'res/snd/*',

test/test_skills_module.py

@@ -97,16 +97,15 @@ def started_hook():
         service = NeonSkillService(alive_hook, started_hook, ready_hook,
                                    error_hook, stopping_hook, config=config,
                                    daemonic=True)
-        from mycroft.configuration import Configuration
+        from neon_core.configuration import Configuration
         self.assertEqual(service.config, Configuration())
-        print(config["skills"])
-        print(service.config['skills'])
         self.assertTrue(all(config['skills'][x] == service.config['skills'][x]
                             for x in config['skills']))
         service.bus = FakeBus()
         service.bus.connected_event = Event()
         service.start()
         started.wait(30)
+        self.assertTrue(service.config['skills']['run_gui_file_server'])
         self.assertIsNotNone(service.http_server)
         self.assertTrue(service.config['skills']['auto_update'])
         install_default.assert_called_once()
@@ -124,6 +123,37 @@ def started_hook():
         stopping_hook.assert_called_once()
         service.join(10)
 
+    @patch("ovos_utils.skills.locations.get_plugin_skills")
+    @patch("ovos_utils.skills.locations.get_skill_directories")
+    def test_get_skill_dirs(self, skill_dirs, plugin_skills):
+        from neon_core.skills.service import NeonSkillService
+
+        test_dir = join(dirname(__file__), "get_skill_dirs_skills")
+        skill_dirs.return_value = [join(test_dir, "extra_dir_1"),
+                                   join(test_dir, "extra_dir_2")]
+        plugin_skills.return_value = ([join(test_dir, "plugins",
+                                            "skill-plugin")],
+                                      ["skill-plugin.neongeckocom"])
+
+        skill_dirs = NeonSkillService()._get_skill_dirs()
+        # listdir doesn't guarantee order, base skill directory order matters
+        self.assertEqual(set(skill_dirs),
+                         {join(test_dir, "plugins", "skill-plugin"),
+                          join(test_dir, "extra_dir_1",
+                               "skill-test-1.neongeckocom"),
+                          join(test_dir, "extra_dir_1",
+                               "skill-test-2.neongeckocom"),
+                          join(test_dir, "extra_dir_1",
+                               "skill-test-3.neongeckocom"),
+                          join(test_dir, "extra_dir_2",
+                               "skill-test-1.neongeckocom")
+                          })
+        self.assertEqual(skill_dirs[0],
+                         join(test_dir, "plugins", "skill-plugin"))
+        self.assertEqual(skill_dirs[-1],
+                         join(test_dir, "extra_dir_2",
+                              "skill-test-1.neongeckocom"))
+
 
 class TestIntentService(unittest.TestCase):
     bus = FakeBus()