Oembed config - oembed_endpoints (matrix-org#2752)

Moved twitter patterns from preview_url_resource to oembed_endpoints

Signed-off-by: Srdjan <srdjan@catalyst.net.nz>

changelog.d/10536.misc

@@ -0,0 +1 @@
+Added `oembed_endpoints` stanza to `homeserver.yaml`, supprted with `OembedConfig` class.

docs/sample_config.yaml

@@ -2605,3 +2605,14 @@ redis:
   # Optional password if configured on the Redis instance
   #
   #password: <secret_password>
+
+
+## Oembed ##
+# A map of globs to API endpoints.
+
+### Twitter.
+"https://publish.twitter.com/oembed":
+  - "https://twitter.com/*/status/*"
+  - "https://*.twitter.com/*/status/*"
+  - "https://twitter.com/*/moments/*"
+  - "https://*.twitter.com/*/moments/*"

synapse/config/_base.pyi

@@ -19,6 +19,7 @@ from synapse.config import (
     logger,
     metrics,
     modules,
+    oembed,
     oidc,
     password_auth_providers,
     push,
@@ -91,6 +92,7 @@ class RootConfig:
     modules: modules.ModulesConfig
     caches: cache.CacheConfig
     federation: federation.FederationConfig
+    oembed: oembed.OembedConfig
 
     config_classes: List = ...
     def __init__(self) -> None: ...

synapse/config/homeserver.py

@@ -30,6 +30,7 @@
 from .logger import LoggingConfig
 from .metrics import MetricsConfig
 from .modules import ModulesConfig
+from .oembed import OembedConfig
 from .oidc import OIDCConfig
 from .password_auth_providers import PasswordAuthProviderConfig
 from .push import PushConfig
@@ -94,5 +95,6 @@ class HomeServerConfig(RootConfig):
         TracerConfig,
         WorkerConfig,
         RedisConfig,
+        OembedConfig,
         ExperimentalConfig,
     ]

synapse/config/oembed.py

@@ -0,0 +1,85 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import re
+from urllib import parse as urlparse
+
+from ._base import Config
+from ._util import validate_config
+
+
+class OembedConfig(Config):
+    section = "oembed"
+
+    def read_config(self, config, **kwargs):
+        # FIXME: oembed_patterns needs sytests
+        self.oembed_patterns = {}
+
+        oembed_endpoints = config.get("oembed_endpoints", {})
+        validate_config(
+            _OEMBED_SCHEMA,
+            oembed_endpoints,
+            ("oembed_endpoints",),
+        )
+        for endpoint, globs in oembed_endpoints.items():
+            for glob in globs:
+                # Convert the glob into a sane regular expression to match against. The
+                # rules followed will be slightly different for the domain portion vs.
+                # the rest.
+                #
+                # 1. The scheme must be one of HTTP / HTTPS (and have no globs).
+                # 2. The domain can have globs, but we limit it to characters that can
+                #    reasonably be a domain part.
+                #    TODO: This does not attempt to handle Unicode domain names.
+                # 3. Other parts allow a glob to be any one, or more, characters.
+                results = urlparse.urlparse(glob)
+
+                # Ensure the scheme does not have wildcards (and is a sane scheme).
+                if results.scheme not in {"http", "https"}:
+                    raise ValueError(
+                        "Insecure oEmbed glob scheme: %s" % (results.scheme,)
+                    )
+
+                pattern = urlparse.urlunparse(
+                    [
+                        results.scheme,
+                        re.escape(results.netloc).replace("\\*", "[a-zA-Z0-9_-]+"),
+                    ]
+                    + [re.escape(part).replace("\\*", ".+") for part in results[2:]]
+                )
+                self.oembed_patterns[re.compile(pattern)] = endpoint
+
+    def generate_config_section(self, config_dir_path, server_name, **kwargs):
+        return """\
+        ## Oembed ##
+        # A map of globs to API endpoints.
+
+        ### Twitter.
+        "https://publish.twitter.com/oembed":
+          - "https://twitter.com/*/status/*"
+          - "https://*.twitter.com/*/status/*"
+          - "https://twitter.com/*/moments/*"
+          - "https://*.twitter.com/*/moments/*"
+        """
+
+
+_HTTPS_URL = "^https://"
+_OEMBED_SCHEMA = {
+    "type": "object",
+    "patternProperties": {
+        _HTTPS_URL: {
+            "type": "array",
+            "items": {"type": "string", "pattern": _HTTPS_URL},
+        }
+    },
+}

synapse/rest/media/v1/preview_url_resource.py

@@ -71,49 +71,6 @@
 
 ONE_HOUR = 60 * 60 * 1000
 
-# A map of globs to API endpoints.
-_oembed_globs = {
-    # Twitter.
-    "https://publish.twitter.com/oembed": [
-        "https://twitter.com/*/status/*",
-        "https://*.twitter.com/*/status/*",
-        "https://twitter.com/*/moments/*",
-        "https://*.twitter.com/*/moments/*",
-        # Include the HTTP versions too.
-        "http://twitter.com/*/status/*",
-        "http://*.twitter.com/*/status/*",
-        "http://twitter.com/*/moments/*",
-        "http://*.twitter.com/*/moments/*",
-    ],
-}
-# Convert the globs to regular expressions.
-_oembed_patterns = {}
-for endpoint, globs in _oembed_globs.items():
-    for glob in globs:
-        # Convert the glob into a sane regular expression to match against. The
-        # rules followed will be slightly different for the domain portion vs.
-        # the rest.
-        #
-        # 1. The scheme must be one of HTTP / HTTPS (and have no globs).
-        # 2. The domain can have globs, but we limit it to characters that can
-        #    reasonably be a domain part.
-        #    TODO: This does not attempt to handle Unicode domain names.
-        # 3. Other parts allow a glob to be any one, or more, characters.
-        results = urlparse.urlparse(glob)
-
-        # Ensure the scheme does not have wildcards (and is a sane scheme).
-        if results.scheme not in {"http", "https"}:
-            raise ValueError("Insecure oEmbed glob scheme: %s" % (results.scheme,))
-
-        pattern = urlparse.urlunparse(
-            [
-                results.scheme,
-                re.escape(results.netloc).replace("\\*", "[a-zA-Z0-9_-]+"),
-            ]
-            + [re.escape(part).replace("\\*", ".+") for part in results[2:]]
-        )
-        _oembed_patterns[re.compile(pattern)] = endpoint
-
 
 @attr.s(slots=True)
 class OEmbedResult:
@@ -144,6 +101,7 @@ def __init__(
         self.clock = hs.get_clock()
         self.filepaths = media_repo.filepaths
         self.max_spider_size = hs.config.max_spider_size
+        self.oembed_patterns = hs.config.oembed_patterns
         self.server_name = hs.hostname
         self.store = hs.get_datastore()
         self.client = SimpleHttpClient(
@@ -377,7 +335,7 @@ def _get_oembed_url(self, url: str) -> Optional[str]:
         Returns:
             A URL to use instead or None if the original URL should be used.
         """
-        for url_pattern, endpoint in _oembed_patterns.items():
+        for url_pattern, endpoint in self.oembed_patterns.items():
             if url_pattern.fullmatch(url):
                 return endpoint