From 3ff7ce07af057868281dec32b2f5508a5b33a710 Mon Sep 17 00:00:00 2001 From: Felix Baron Date: Mon, 25 Oct 2021 22:06:27 +0200 Subject: [PATCH 1/4] Reworked youtube search request --- requirements.txt | 1 + src/jarvis/jarvis/settings.py | 4 +-- .../jarvis/skills/collection/browser.py | 33 ++++++++++++------- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/requirements.txt b/requirements.txt index a9327dd4..d3966a78 100644 --- a/requirements.txt +++ b/requirements.txt @@ -49,3 +49,4 @@ wikipedia==1.4.0 wolframalpha==3.0.1 word2number==1.1 xmltodict==0.12.0 +glom==20.11.0 diff --git a/src/jarvis/jarvis/settings.py b/src/jarvis/jarvis/settings.py index 98f5a0c6..9e5432db 100644 --- a/src/jarvis/jarvis/settings.py +++ b/src/jarvis/jarvis/settings.py @@ -25,7 +25,7 @@ ROOT_LOG_CONF = { 'version': 1, 'root': { - 'level': 'INFO', + 'level': 'DEBUG', 'handlers': ['file'], }, 'handlers': { @@ -68,7 +68,7 @@ DEFAULT_GENERAL_SETTINGS = { 'assistant_name': 'Jarvis', - 'input_mode': InputMode.TEXT.value, + 'input_mode': InputMode.VOICE.value, 'response_in_speech': False, } diff --git a/src/jarvis/jarvis/skills/collection/browser.py b/src/jarvis/jarvis/skills/collection/browser.py index 11d35677..1285c12e 100644 --- a/src/jarvis/jarvis/skills/collection/browser.py +++ b/src/jarvis/jarvis/skills/collection/browser.py @@ -27,8 +27,10 @@ import urllib.request import subprocess import webbrowser +import json from bs4 import BeautifulSoup as bs +from glom import glom, Coalesce from jarvis.skills.skill import AssistantSkill @@ -68,18 +70,26 @@ def open_in_youtube(cls, voice_transcript, skill): tags = cls.extract_tags(voice_transcript, skill['tags']) for tag in tags: reg_ex = re.search(tag + ' (.*)', voice_transcript) - try: - if reg_ex: - search_text = reg_ex.group(1) - base = "https://www.youtube.com/results?search_query={0}&orderby=viewCount" - r = requests.get(base.format(search_text.replace(' ', '+'))) - page = r.text - soup = bs(page, 'html.parser') - vids = soup.findAll('a', attrs={'class': 'yt-uix-tile-link'}) - video = 'https://www.youtube.com' + vids[0]['href'] + "&autoplay=1" - cls.console(info_log="Play Youtube video: {0}".format(video)) - subprocess.Popen(["python", "-m", "webbrowser", "-t", video], stdout=subprocess.PIPE, shell=False) + search_text = reg_ex.group(1) + base = f"https://www.youtube.com/results?search_query={search_text.replace(' ', '+')}&orderby=viewCount" + r = requests.get(base) + page = r.text + reg_ex = re.search("var ytInitialData = (.*);<\/script>", page) + yt_initial_data = json.loads(reg_ex.group(1)) + yt_spec = ('contents.twoColumnSearchResultsRenderer.primaryContents.sectionListRenderer.contents', [ + Coalesce(('itemSectionRenderer.contents', [{ + 'url': Coalesce('radioRenderer.navigationEndpoint.commandMetadata.webCommandMetadata.url', + 'videoRenderer.navigationEndpoint.commandMetadata.webCommandMetadata.url', default=None), + 'title': Coalesce("radioRenderer.title.simpleText", + "videoRenderer.title.accessibility.accessibilityData.label", default="Unknown") + }]), default=None), + ], lambda x: x[0][0]) + # glom with yt_specs returns a nested result-list, lamnda-func at the end selects the top-result + yt_topresult = glom(yt_initial_data, yt_spec) + video = f"https://youtube.com{yt_topresult['url']}" + cls.console(info_log=f"Now playing: {yt_topresult['title']}") + subprocess.Popen(["python", "-m", "webbrowser", "-t", video], stdout=subprocess.PIPE, shell=False) except Exception as e: cls.console(error_log="Error with the execution of skill with message {0}".format(e)) cls.response("I can't find what do you want in Youtube..") @@ -175,4 +185,3 @@ def _search_on_google(cls, term): except Exception as e: cls.console(error_log="Error with the execution of skill with message {0}".format(e)) cls.response("Sorry I faced an issue with google search") - From 4e1a5570086a605d9988634da2d07c6438ac7ecb Mon Sep 17 00:00:00 2001 From: Felix Baron Date: Mon, 25 Oct 2021 23:30:03 +0200 Subject: [PATCH 2/4] improved results for complex search terms added error-message if no results were found (missing url) undid test-format for requests --- src/jarvis/jarvis/skills/collection/browser.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/jarvis/jarvis/skills/collection/browser.py b/src/jarvis/jarvis/skills/collection/browser.py index 1285c12e..87cf3b34 100644 --- a/src/jarvis/jarvis/skills/collection/browser.py +++ b/src/jarvis/jarvis/skills/collection/browser.py @@ -72,8 +72,8 @@ def open_in_youtube(cls, voice_transcript, skill): reg_ex = re.search(tag + ' (.*)', voice_transcript) try: search_text = reg_ex.group(1) - base = f"https://www.youtube.com/results?search_query={search_text.replace(' ', '+')}&orderby=viewCount" - r = requests.get(base) + base = "https://www.youtube.com/results?search_query={0}&orderby=viewCount" + r = requests.get(base.format(search_text.replace(' ', '+'))) page = r.text reg_ex = re.search("var ytInitialData = (.*);<\/script>", page) yt_initial_data = json.loads(reg_ex.group(1)) @@ -84,14 +84,16 @@ def open_in_youtube(cls, voice_transcript, skill): 'title': Coalesce("radioRenderer.title.simpleText", "videoRenderer.title.accessibility.accessibilityData.label", default="Unknown") }]), default=None), - ], lambda x: x[0][0]) - # glom with yt_specs returns a nested result-list, lamnda-func at the end selects the top-result - yt_topresult = glom(yt_initial_data, yt_spec) - video = f"https://youtube.com{yt_topresult['url']}" - cls.console(info_log=f"Now playing: {yt_topresult['title']}") + ], lambda x: [elem for elem in x[0] if elem['url'] != None]) + # glom with yt_specs returns a nested result-list, lamnda-func filters the list + yt_results = glom(yt_initial_data, yt_spec) + if not yt_results: + raise Exception(f"No YT-video was found for '{search_text}'.") + video = f"https://youtube.com{yt_results[0]['url']}" + cls.console(info_log=f"Now playing: {yt_results[0]['title']}") subprocess.Popen(["python", "-m", "webbrowser", "-t", video], stdout=subprocess.PIPE, shell=False) except Exception as e: - cls.console(error_log="Error with the execution of skill with message {0}".format(e)) + cls.console(error_log="Error with the execution of skill with message: {0}".format(e)) cls.response("I can't find what do you want in Youtube..") @classmethod From be783b342b44ac800fdfb7e2946519fa497c44d4 Mon Sep 17 00:00:00 2001 From: Felix Baron Date: Tue, 26 Oct 2021 21:32:35 +0200 Subject: [PATCH 3/4] setting back to default --- src/jarvis/jarvis/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jarvis/jarvis/settings.py b/src/jarvis/jarvis/settings.py index 9e5432db..98f5a0c6 100644 --- a/src/jarvis/jarvis/settings.py +++ b/src/jarvis/jarvis/settings.py @@ -25,7 +25,7 @@ ROOT_LOG_CONF = { 'version': 1, 'root': { - 'level': 'DEBUG', + 'level': 'INFO', 'handlers': ['file'], }, 'handlers': { @@ -68,7 +68,7 @@ DEFAULT_GENERAL_SETTINGS = { 'assistant_name': 'Jarvis', - 'input_mode': InputMode.VOICE.value, + 'input_mode': InputMode.TEXT.value, 'response_in_speech': False, } From f110c02dcaa0976bc3ce8a1cdb8ce1f4493cd179 Mon Sep 17 00:00:00 2001 From: Felix Baron Date: Sat, 30 Oct 2021 17:57:42 +0200 Subject: [PATCH 4/4] Jarvisoutput instead of Exeption + Errormsg --- src/jarvis/jarvis/skills/collection/browser.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/jarvis/jarvis/skills/collection/browser.py b/src/jarvis/jarvis/skills/collection/browser.py index 87cf3b34..22a332ba 100644 --- a/src/jarvis/jarvis/skills/collection/browser.py +++ b/src/jarvis/jarvis/skills/collection/browser.py @@ -73,7 +73,7 @@ def open_in_youtube(cls, voice_transcript, skill): try: search_text = reg_ex.group(1) base = "https://www.youtube.com/results?search_query={0}&orderby=viewCount" - r = requests.get(base.format(search_text.replace(' ', '+'))) + r = requests.get(base.format(search_text.replace(' ', '+'))) page = r.text reg_ex = re.search("var ytInitialData = (.*);<\/script>", page) yt_initial_data = json.loads(reg_ex.group(1)) @@ -82,18 +82,19 @@ def open_in_youtube(cls, voice_transcript, skill): 'url': Coalesce('radioRenderer.navigationEndpoint.commandMetadata.webCommandMetadata.url', 'videoRenderer.navigationEndpoint.commandMetadata.webCommandMetadata.url', default=None), 'title': Coalesce("radioRenderer.title.simpleText", - "videoRenderer.title.accessibility.accessibilityData.label", default="Unknown") + "videoRenderer.title.accessibility.accessibilityData.label", default="Unknown") }]), default=None), ], lambda x: [elem for elem in x[0] if elem['url'] != None]) # glom with yt_specs returns a nested result-list, lamnda-func filters the list yt_results = glom(yt_initial_data, yt_spec) if not yt_results: - raise Exception(f"No YT-video was found for '{search_text}'.") - video = f"https://youtube.com{yt_results[0]['url']}" - cls.console(info_log=f"Now playing: {yt_results[0]['title']}") - subprocess.Popen(["python", "-m", "webbrowser", "-t", video], stdout=subprocess.PIPE, shell=False) + cls.response(f"I could not find a video searching for {search_text}") + else: + video = f"https://youtube.com{yt_results[0]['url']}" + cls.console(info_log=f"Now playing: {yt_results[0]['title']}") + subprocess.Popen(["python", "-m", "webbrowser", "-t", video], stdout=subprocess.PIPE, shell=False) except Exception as e: - cls.console(error_log="Error with the execution of skill with message: {0}".format(e)) + cls.console(error_log="Error with the execution of skill with message {0}".format(e)) cls.response("I can't find what do you want in Youtube..") @classmethod