plugins: handle URL callbacks in new rule system

sopel/bot.py

@@ -420,7 +420,6 @@ def remove_plugin(self, plugin, callables, jobs, shutdowns, urls):
         self._rules_manager.unregister_plugin(name)
         self._scheduler.unregister_plugin(name)
         self.unregister_shutdowns(shutdowns)
-        self.unregister_urls(urls)
 
         # remove plugin from registry
         del self._plugins[name]
@@ -573,22 +572,21 @@ def unregister_shutdowns(self, shutdowns):
 
     def register_urls(self, urls):
         for func in urls:
-            for regex in func.url_regex:
-                self.register_url_callback(regex, func)
-                callable_name = getattr(func, "__name__", 'UNKNOWN')
-                LOGGER.debug(
-                    'URL Callback added "%s" for URL pattern "%s"',
-                    callable_name,
-                    regex)
-
-    def unregister_urls(self, urls):
-        if "url_callbacks" in self.memory:
-            for func in urls:
-                regexes = func.url_regex
-                for regex in regexes:
-                    if func == self.memory['url_callbacks'].get(regex):
-                        self.unregister_url_callback(regex, func)
-                        LOGGER.debug('URL Callback unregistered: %r', regex)
+            url_regex = getattr(func, 'url_regex', [])
+            url_lazy_loaders = getattr(func, 'url_lazy_loaders', None)
+
+            if url_regex:
+                rule = plugin_rules.URLCallback.from_callable(
+                    self.settings, func)
+                self._rules_manager.register_url_callback(rule)
+
+            if url_lazy_loaders:
+                try:
+                    rule = plugin_rules.URLCallback.from_callable_lazy(
+                        self.settings, func)
+                    self._rules_manager.register_url_callback(rule)
+                except plugins.exceptions.PluginError as err:
+                    LOGGER.error('Cannot register URL callback: %s', err)
 
     @deprecated(
         reason="Replaced by `say` method.",

sopel/config/core_section.py

@@ -20,6 +20,8 @@
 """Default prefix used for commands."""
 COMMAND_DEFAULT_HELP_PREFIX = '.'
 """Default help prefix used in commands' usage messages."""
+URL_DEFAULT_SCHEMES = ['http', 'https', 'ftp']
+"""Default URL schemes allowed for URLs."""
 
 
 def _find_certs():
@@ -188,7 +190,7 @@ class CoreSection(StaticSection):
     auto_url_schemes = ListAttribute(
         'auto_url_schemes',
         strip=True,
-        default=['http', 'https', 'ftp'])
+        default=URL_DEFAULT_SCHEMES)
     """List of URL schemes that will trigger URL callbacks.
 
     :default: ``['http', 'https', 'ftp']``

sopel/irc/__init__.py

@@ -213,7 +213,11 @@ def on_message(self, message):
         """
         self.last_raw_line = message
 
-        pretrigger = PreTrigger(self.nick, message)
+        pretrigger = PreTrigger(
+            self.nick,
+            message,
+            url_schemes=self.settings.core.auto_url_schemes,
+        )
         if all(cap not in self.enabled_capabilities for cap in ['account-tag', 'extended-join']):
             pretrigger.tags.pop('account', None)
 
@@ -262,7 +266,8 @@ def on_message_sent(self, raw):
 
             pretrigger = PreTrigger(
                 self.nick,
-                ":{0}!{1}@{2} {3}".format(self.nick, self.user, host, raw)
+                ":{0}!{1}@{2} {3}".format(self.nick, self.user, host, raw),
+                url_schemes=self.settings.core.auto_url_schemes,
             )
             self.dispatch(pretrigger)
 

sopel/loader.py

@@ -62,9 +62,9 @@ def clean_callable(func, config):
         func.global_rate = getattr(func, 'global_rate', 0)
         func.unblockable = getattr(func, 'unblockable', False)
 
-    if not is_triggerable(func):
-        # Adding the remaining default attributes below is potentially confusing
-        # to other code (and a waste of memory) for non-triggerable functions.
+    if not is_triggerable(func) and not is_url_callback(func):
+        # Adding the remaining default attributes below is potentially
+        # confusing to other code (and a waste of memory) for jobs.
         return
 
     func.echo = getattr(func, 'echo', False)
@@ -152,6 +152,7 @@ def is_limitable(obj):
         'nickname_commands',
         'action_commands',
         'url_regex',
+        'url_lazy_loaders',
     )
     allowed = any(hasattr(obj, attr) for attr in allowed_attrs)
 
@@ -173,10 +174,12 @@ def is_triggerable(obj):
 
         Many of the decorators defined in :mod:`sopel.plugin` make the
         decorated function a triggerable object.
+
     """
     forbidden_attrs = (
         'interval',
         'url_regex',
+        'url_lazy_loaders',
     )
     forbidden = any(hasattr(obj, attr) for attr in forbidden_attrs)
 
@@ -195,6 +198,35 @@ def is_triggerable(obj):
     return allowed and not forbidden
 
 
+def is_url_callback(obj):
+    """Check if ``obj`` can handle a URL callback.
+
+    :param obj: any :term:`function` to check
+    :return: ``True`` if ``obj`` can handle a URL callback
+
+    A URL callback handler is a callable that will be used by the bot to
+    handle a particular URL in an IRC message.
+
+    .. seealso::
+
+        Both :func:`sopel.plugin.url` :func:`sopel.plugin.url_lazy` make the
+        decorated function a URL callback handler.
+
+    """
+    forbidden_attrs = (
+        'interval',
+    )
+    forbidden = any(hasattr(obj, attr) for attr in forbidden_attrs)
+
+    allowed_attrs = (
+        'url_regex',
+        'url_lazy_loaders',
+    )
+    allowed = any(hasattr(obj, attr) for attr in allowed_attrs)
+
+    return allowed and not forbidden
+
+
 def clean_module(module, config):
     callables = []
     shutdowns = []
@@ -210,7 +242,7 @@ def clean_module(module, config):
             elif hasattr(obj, 'interval'):
                 clean_callable(obj, config)
                 jobs.append(obj)
-            elif hasattr(obj, 'url_regex'):
+            elif is_url_callback(obj):
                 clean_callable(obj, config)
                 urls.append(obj)
     return callables, jobs, shutdowns, urls

sopel/modules/bugzilla.py

@@ -14,11 +14,9 @@
 import requests
 import xmltodict
 
+from sopel import plugin, plugins
 from sopel.config.types import ListAttribute, StaticSection
-from sopel.module import rule
 
-
-regex = None
 LOGGER = logging.getLogger(__name__)
 
 
@@ -42,45 +40,44 @@ def configure(config):
 
 
 def setup(bot):
-    global regex
     bot.config.define_section('bugzilla', BugzillaSection)
 
-    if not bot.config.bugzilla.domains:
-        return
 
-    domains = '|'.join(bot.config.bugzilla.domains)
-    regex = re.compile((r'https?://(%s)'
-                        r'(/show_bug.cgi\?\S*?)'
-                        r'(id=\d+)')
-                       % domains)
-    bot.register_url_callback(regex, show_bug)
+def _bugzilla_loader(settings):
+    if not settings.bugzilla.domains:
+        raise plugins.exceptions.PluginSettingsError(
+            'Bugzilla URL callback requires '
+            '"bugzilla.domains" to be configured; check your config file.')
 
+    domain_pattern = '|'.join(
+        re.escape(domain)
+        for domain in settings.bugzilla.domains)
 
-def shutdown(bot):
-    bot.unregister_url_callback(regex, show_bug)
+    pattern = (
+        r'https?://(%s)'
+        r'(/show_bug.cgi\?\S*?)'
+        r'(id=\d+).*'
+    ) % domain_pattern
 
+    return [re.compile(pattern)]
 
-@rule(r'.*https?://(\S+?)'
-      r'(/show_bug.cgi\?\S*?)'
-      r'(id=\d+).*')
+
+@plugin.url_lazy(_bugzilla_loader)
+@plugin.output_prefix('[BUGZILLA] ')
 def show_bug(bot, trigger, match=None):
     """Show information about a Bugzilla bug."""
-    match = match or trigger
-    domain = match.group(1)
-    if domain not in bot.config.bugzilla.domains:
-        return
-    url = 'https://%s%sctype=xml&%s' % match.groups()
+    url = 'https://%s%sctype=xml&%s' % trigger.groups()
     data = requests.get(url).content
     bug = xmltodict.parse(data).get('bugzilla').get('bug')
     error = bug.get('@error', None)  # error="NotPermitted"
 
     if error:
         LOGGER.warning('Bugzilla error: %s' % error)
-        bot.say('[BUGZILLA] Unable to get information for '
+        bot.say('Unable to get information for '
                 'linked bug (%s)' % error)
         return
 
-    message = ('[BUGZILLA] %s | Product: %s | Component: %s | Version: %s | ' +
+    message = ('%s | Product: %s | Component: %s | Version: %s | ' +
                'Importance: %s |  Status: %s | Assigned to: %s | ' +
                'Reported: %s | Modified: %s')
 

sopel/modules/url.py

@@ -277,11 +277,11 @@ def check_callbacks(bot, url):
 
     :param bot: Sopel instance
     :param str url: URL to check
-    :return: True if ``url`` is excluded or matches any URL Callback pattern
+    :return: True if ``url`` is excluded or matches any URL callback pattern
 
     This function looks at the ``bot.memory`` for ``url_exclude`` patterns and
     it returns ``True`` if any matches the given ``url``. Otherwise, it looks
-    at the ``bot``'s URL Callback patterns, and it returns ``True`` if any
+    at the ``bot``'s URL callback patterns, and it returns ``True`` if any
     matches, ``False`` otherwise.
 
     .. seealso::
@@ -297,7 +297,11 @@ def check_callbacks(bot, url):
     """
     # Check if it matches the exclusion list first
     matched = any(regex.search(url) for regex in bot.memory['url_exclude'])
-    return matched or any(bot.search_url_callbacks(url))
+    return (
+        matched or
+        any(bot.search_url_callbacks(url)) or
+        bot.rules.check_url_callback(bot, url)
+    )
 
 
 def find_title(url, verify=True):

sopel/modules/wikipedia.py

@@ -13,7 +13,7 @@
 from requests import get
 
 from sopel.config.types import StaticSection, ValidatedAttribute
-from sopel.module import commands, example, NOLIMIT, url
+from sopel.module import commands, example, NOLIMIT, output_prefix, url
 from sopel.tools.web import quote, unquote
 
 try:  # TODO: Remove fallback when dropping py2
@@ -133,9 +133,9 @@ def say_snippet(bot, trigger, server, query, show_url=True):
         snippet = mw_snippet(server, query)
     except KeyError:
         if show_url:
-            bot.say("[WIKIPEDIA] Error fetching snippet for \"{}\".".format(page_name))
+            bot.say("Error fetching snippet for \"{}\".".format(page_name))
         return
-    msg = '[WIKIPEDIA] {} | "{}"'.format(page_name, snippet)
+    msg = '{} | "{}"'.format(page_name, snippet)
     msg_url = msg + ' | https://{}/wiki/{}'.format(server, query)
     if msg_url == trigger:  # prevents triggering on another instance of Sopel
         return
@@ -166,10 +166,10 @@ def say_section(bot, trigger, server, query, section):
 
     snippet = mw_section(server, query, section)
     if not snippet:
-        bot.say("[WIKIPEDIA] Error fetching section \"{}\" for page \"{}\".".format(section, page_name))
+        bot.say("Error fetching section \"{}\" for page \"{}\".".format(section, page_name))
         return
 
-    msg = '[WIKIPEDIA] {} - {} | "{}"'.format(page_name, section.replace('_', ' '), snippet)
+    msg = '{} - {} | "{}"'.format(page_name, section.replace('_', ' '), snippet)
     bot.say(msg)
 
 
@@ -217,6 +217,7 @@ def mw_section(server, query, section):
 
 # Matches a wikipedia page (excluding spaces and #, but not /File: links), with a separate optional field for the section
 @url(r'https?:\/\/([a-z]+\.wikipedia\.org)\/wiki\/((?!File\:)[^ #]+)#?([^ ]*)')
+@output_prefix('[WIKIPEDIA] ')
 def mw_info(bot, trigger, match=None):
     """Retrieves and outputs a snippet of the linked page."""
     if match.group(3):
@@ -230,6 +231,7 @@ def mw_info(bot, trigger, match=None):
 
 @commands('w', 'wiki', 'wik')
 @example('.w San Francisco')
+@output_prefix('[WIKIPEDIA] ')
 def wikipedia(bot, trigger):
     lang = bot.config.wikipedia.default_lang