Fetch images when previewing Twitter URLs. (#11985)

By including "bot" in the User-Agent, which some sites use to decide whether to include additional Open Graph information.
matrix-org · Feb 22, 2022 · 0661716 · 0661716
1 parent 79c18e0
commit 0661716
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 4 deletions.
diff --git a/changelog.d/11985.feature b/changelog.d/11985.feature
@@ -0,0 +1 @@
+Fetch images when previewing Twitter URLs. Contributed by @AndrewRyanChama.
diff --git a/synapse/res/providers.json b/synapse/res/providers.json
@@ -5,13 +5,11 @@
         "endpoints": [
             {
                 "schemes": [
-                    "https://twitter.com/*/status/*",
-                    "https://*.twitter.com/*/status/*",
                     "https://twitter.com/*/moments/*",
                     "https://*.twitter.com/*/moments/*"
                 ],
                 "url": "https://publish.twitter.com/oembed"
             }
         ]
     }
-]
+]
diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py
@@ -402,7 +402,15 @@ async def _download_url(self, url: str, output_stream: BinaryIO) -> DownloadResu
                 url,
                 output_stream=output_stream,
                 max_size=self.max_spider_size,
-                headers={"Accept-Language": self.url_preview_accept_language},
+                headers={
+                    b"Accept-Language": self.url_preview_accept_language,
+                    # Use a custom user agent for the preview because some sites will only return
+                    # Open Graph metadata to crawler user agents. Omit the Synapse version
+                    # string to avoid leaking information.
+                    b"User-Agent": [
+                        "Synapse (bot; +https://github.com/matrix-org/synapse)"
+                    ],
+                },
                 is_allowed_content_type=_is_previewable,
             )
         except SynapseError:
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Fetch images when previewing Twitter URLs. Contributed by @AndrewRyanChama.