Fix some 2.19 regressions and improve twitter and github spouts

NEWS.md

@@ -23,6 +23,8 @@
 - Tag colour can be now changed using keyboard. ([#1335](https://github.com/fossar/selfoss/pull/1335))
 - YouTube spout now supports all YouTube URLs that provide feeds. ([#1273](https://github.com/fossar/selfoss/issues/1273))
 - Add `open_in_background_tab` option to try to make <kbd>v</kbd> shortcut open articles in a background tab ([does not work in Chromium-based browsers](https://crbug.com/431335)). ([#1354](https://github.com/fossar/selfoss/pull/1354))
+- GitHub sources now include author. ([#1367](https://github.com/fossar/selfoss/pull/1367))
+- Twitter sources now indicate author using the author field rather than including in the title. ([#1367](https://github.com/fossar/selfoss/pull/1367))
 - Translations into several new languages were added:
   - English (United Kingdom): `en-GB`
   - French (Canada): `fr-CA`
@@ -57,6 +59,7 @@
   - `GET /login` endpoint, use `POST /login`. ([#1360](https://github.com/fossar/selfoss/pull/1360))
   - `GET /logout` was deprecated in favour of newly introduced (*API 4.1.0*) `DELETE /api/session/current`. ([#1360](https://github.com/fossar/selfoss/pull/1360))
   - `POST /source/delete/:id` in favour of `DELETE /source/:id`. ([#1360](https://github.com/fossar/selfoss/pull/1360))
+- *API 6.0.0*: Makes the `author` field `null` when an item author is not known ([#1367](https://github.com/fossar/selfoss/pull/1367))
 
 ### Customization changes
 - `selfoss.shares.register` was removed. Instead you should set `selfoss.customSharers` to an object of *sharer* objects. The `action` callback is now expected to open a window on its own, instead of returning a URL. A label and a HTML code of an icon (you can use a `<img>` tag, inline `<svg>`, emoji, etc.) are now expected.

assets/js/requests/sources.js

@@ -5,8 +5,11 @@ import * as ajax from '../helpers/ajax';
  */
 export function update(id, values) {
     return ajax.post(`source/${id}`, {
-        body: new URLSearchParams(values),
-        failOnHttpErrors: false
+        headers: {
+            'content-type': 'application/json; charset=utf-8'
+        },
+        body: JSON.stringify(values),
+        failOnHttpErrors: false,
     }).promise
         .then(ajax.rejectUnless(response => response.ok || response.status === 400))
         .then(response => response.json());

assets/js/templates/Item.jsx

@@ -14,6 +14,11 @@ import { Direction } from '../helpers/navigation';
 import { useSharers } from '../sharers';
 import Lightbox from 'yet-another-react-lightbox';
 
+// TODO: do the search highlights client-side
+function reHighlight(text) {
+    return text.split(/<span class="found">(.+?)<\/span>/).map((n, i) => i % 2 == 0 ? n : <span className="found">{n}</span>);
+}
+
 function setupLightbox({
     element,
     setSlides,
@@ -461,13 +466,13 @@ export default function Item({ currentTime, item, selected, expanded, setNavExpa
                 to={sourceLink}
                 onClick={preventDefaultOnSmartphone}
             >
-                {sourcetitle}
+                {reHighlight(sourcetitle)}
             </Link>
 
             <span className="entry-separator">•</span>
 
             {/* author */}
-            {author.trim() !== '' ?
+            {author !== null ?
                 <React.Fragment>
                     <span className="entry-author">{author}</span>
                     <span className="entry-separator">•</span>

assets/js/templates/Source.jsx

@@ -63,10 +63,7 @@ function handleSave({
     setSourceActionLoading(true);
 
     const newSource = source;
-    const { id, tags, params, ...restSource } = source;
-
-    // Build params for the API request.
-    let values = Object.entries({ ...restSource, ...params });
+    const { id, tags, filter, params, ...restSource } = source;
 
     // Make tags into a list.
     const tagsList = tags
@@ -76,7 +73,12 @@ function handleSave({
             .filter((tag) => tag !== '')
         : [];
 
-    tagsList.forEach((tag) => values.push(['tags[]', tag]));
+    const values = {
+        ...restSource,
+        ...params,
+        tags: tagsList,
+        filter: filter || null,
+    };
 
     sourceRequests
         .update(id, values)

docs/api-description.json

@@ -3,7 +3,7 @@
   "servers": [],
   "info": {
     "description": "You can access selfoss by using the same backend as selfoss user interface: The RESTful HTTP JSON API. There are a few urls where you can get information from selfoss and some for updating data. Assume you want all tags for rendering this in your own app. You have to make an HTTP GET call on the url /tags:\n\n```\nGET http://yourselfossurl.com/tags\n```\nThe result is following JSON formatted response (in this example two tags “blog” and “deviantart” are available:\n\n```\n[{\"tag\":\"blog\",\"color\":\"#251f10\",\"unread\":\"1\"},\n{\"tag\":\"deviantart\",\"color\":\"#e78e5c\",\"unread\":\"0\"}]\n```\n\nFollowing docs shows you which calls are possible and which response you can expect.",
-    "version": "5.0.0",
+    "version": "6.0.0",
     "title": "selfoss"
   },
   "tags": [
@@ -954,6 +954,12 @@
             "type": "string",
             "example": "\n<p>Das 1-GBit/s-Angebot Google Fiber kommt nach Austin, die Hauptstadt des US-Bundesstaates Texas..."
           },
+          "author": {
+            "description": "Name/e-mail of the author of the item/article/tweet, when available.",
+            "type": "string",
+            "nullable": true,
+            "example": "William Miller"
+          },
           "unread": {
             "description": "true when the article is marked as unread, false when the article is marked as read",
             "type": "boolean",

docs/content/docs/administration/requirements.md

@@ -6,7 +6,7 @@ extra.show_next = true
 
 selfoss is not a hosted service. It has to be installed on your own web server. This web server must fulfil the following requirements (which are available from most providers)
 
-* PHP 5.6 or higher with the `php-gd` and `php-http` extensions enabled. Some spouts may also require `curl` or `mbstring` extensions. The `php-imagick` extension is required if you want selfoss to support SVG site icons.
+* PHP 5.6 or higher with the `php-gd` and `php-http` extensions enabled. Some spouts may also require `curl`, `mbstring` or `tidy` extensions. The `php-imagick` extension is required if you want selfoss to support SVG site icons.
 * MySQL 5.5.3 or higher, PostgreSQL, or SQLite
 * Apache web server (nginx and Lighttpd also possible)
 

docs/content/docs/customization/api.md

@@ -11,7 +11,7 @@ Since selfoss 2.19, the API follows [semantic versioning](https://semver.org/) d
 
 | selfoss | API |
 |---|---|
-| [2.19] | [5.0.0] |
+| [2.19] | [6.0.0] |
 
 [2.19]: https://github.com/fossar/selfoss/releases/tag/2.19
-[5.0.0]: https://app.swaggerhub.com/apis-docs/jtojnar/selfoss/5.0.0
+[6.0.0]: https://app.swaggerhub.com/apis-docs/jtojnar/selfoss/6.0.0

flake.nix

@@ -36,6 +36,7 @@
         # Create a PHP package from the selected PHP package, with some extra extensions enabled.
         php = phps.packages.${system}.${matrix.phpPackage}.withExtensions ({ enabled, all }: with all; enabled ++ [
           imagick
+          tidy
         ]);
 
         # Create a Python package with some extra packages installed.

src/constants.php

@@ -7,4 +7,4 @@
 // independent of selfoss version
 // needs to be bumped each time public API is changed (follows semver)
 // keep in sync with docs/api-description.json
-const SELFOSS_API_VERSION = '5.0.0';
+const SELFOSS_API_VERSION = '6.0.0';

src/controllers/Opml/Import.php

@@ -71,9 +71,8 @@ public function add() {
             }
 
             $subs = false;
+            $previousUseErrors = libxml_use_internal_errors(true);
             try {
-                $previousUseErrors = libxml_use_internal_errors(true);
-
                 $subs = simplexml_load_file($opml['tmp_name']);
 
                 if ($subs === false) {

src/daos/mysql/Database.php

@@ -287,6 +287,14 @@ public function __construct(DatabaseConnection $connection, Logger $logger) {
             $this->exec('INSERT INTO ' . $this->connection->getTableNamePrefix() . 'version (version) VALUES (13)');
             $this->commit();
         }
+        if ($version < 14) {
+            $this->logger->debug('Upgrading database schema to version 14');
+
+            $this->beginTransaction();
+            $this->exec('UPDATE ' . $this->connection->getTableNamePrefix() . "items SET author = NULL WHERE author = ''");
+            $this->exec('INSERT INTO ' . $this->connection->getTableNamePrefix() . 'version (version) VALUES (14)');
+            $this->commit();
+        }
     }
 
     /**

src/daos/pgsql/Database.php

@@ -265,6 +265,14 @@ public function __construct(DatabaseConnection $connection, Logger $logger) {
             $this->exec('INSERT INTO version (version) VALUES (13)');
             $this->commit();
         }
+        if ($version < 14) {
+            $this->logger->debug('Upgrading database schema to version 14');
+
+            $this->beginTransaction();
+            $this->exec("UPDATE items SET author = NULL WHERE author = ''");
+            $this->exec('INSERT INTO version (version) VALUES (14)');
+            $this->commit();
+        }
     }
 
     /**

src/daos/sqlite/Database.php

@@ -283,6 +283,14 @@ public function __construct(DatabaseConnection $connection, Logger $logger) {
             $this->exec('INSERT INTO version (version) VALUES (13)');
             $this->commit();
         }
+        if ($version < 14) {
+            $this->logger->debug('Upgrading database schema to version 14');
+
+            $this->beginTransaction();
+            $this->exec("UPDATE items SET author = NULL WHERE author = ''");
+            $this->exec('INSERT INTO version (version) VALUES (14)');
+            $this->commit();
+        }
     }
 
     /**

src/helpers/ContentLoader.php

@@ -203,9 +203,6 @@ public function fetch($source) {
                     continue;
                 }
 
-                // sanitize author
-                $author = $this->sanitizeField($item->getAuthor() ?: '');
-
                 $this->logger->debug('item content sanitized');
 
                 $newItem = [
@@ -215,7 +212,7 @@ public function fetch($source) {
                     'datetime' => $itemDate->format('Y-m-d H:i:s'),
                     'uid' => $item->getId(),
                     'link' => htmLawed($item->getLink(), ['deny_attribute' => '*', 'elements' => '-*']),
-                    'author' => $author,
+                    'author' => $item->getAuthor(),
                     'thumbnail' => null,
                     'icon' => null,
                 ];

src/helpers/FeedReader.php

@@ -62,7 +62,8 @@ public function load($url) {
             // save fetched items
             'items' => $this->simplepie->get_items(),
             'htmlUrl' => htmlspecialchars_decode((string) $this->simplepie->get_link(), ENT_COMPAT), // SimplePie sanitizes URLs
-            'title' => $this->simplepie->get_title(),
+            // Atom feeds can contain HTML in titles, strip tags and convert to text.
+            'title' => htmlspecialchars_decode(strip_tags($this->simplepie->get_title())),
         ];
     }
 

src/spouts/github/commits.php

@@ -109,7 +109,7 @@ public function getItems() {
             $link = $item['html_url'];
             // Appears to be ISO 8601.
             $date = new \DateTimeImmutable($item['commit']['author']['date']);
-            $author = null;
+            $author = $item['commit']['author']['name'];
 
             yield new Item(
                 $id,

src/spouts/rss/feed.php

@@ -127,11 +127,14 @@ public function getItems() {
     private function getAuthorString(SimplePie\Item $item) {
         $author = $item->get_author();
         if (isset($author)) {
+            // Both are sanitized using SimplePie::CONSTRUCT_TEXT
+            // so they are plain text strings with escaped HTML special characters.
             $name = $author->get_name();
-            if (isset($name)) {
+            $email = $author->get_email();
+            if ($name !== null) {
                 return htmlspecialchars_decode($name);
-            } else {
-                return htmlspecialchars_decode((string) $author->get_email());
+            } elseif ($email !== null) {
+                return htmlspecialchars_decode($author->get_email());
             }
         }
 

src/spouts/twitter/usertimeline.php

@@ -195,16 +195,22 @@ public function getHtmlUrl() {
      */
     public function getItems() {
         foreach ($this->items as $item) {
+            $author = $item->user->name;
+            $targetItem = $item;
+            if (isset($item->retweeted_status)) {
+                $targetItem = $item->retweeted_status;
+                $author .= ' (RT ' . $targetItem->user->name . ')';
+            }
+
             $id = $item->id_str;
-            $title = $this->getTweetTitle($item);
-            $content = $this->getContent($item);
-            $thumbnail = $this->getThumbnail($item);
-            $icon = $this->getTweetIcon($item);
+            $title = $this->getTweetTitle($targetItem);
+            $content = $this->getContent($targetItem);
+            $thumbnail = $this->getThumbnail($targetItem);
+            $icon = $this->getTweetIcon($targetItem);
             $link = 'https://twitter.com/' . $item->user->screen_name . '/status/' . $item->id_str;
             // Format of `created_at` field not specified, looks US-centric.
             // https://developer.twitter.com/en/docs/twitter-api/v1/data-dictionary/object-model/tweet
             $date = new \DateTimeImmutable($item->created_at);
-            $author = null;
 
             yield new Item(
                 $id,
@@ -223,14 +229,8 @@ public function getItems() {
      * @return string
      */
     private function getTweetTitle(stdClass $item) {
-        $rt = '';
-        if (isset($item->retweeted_status)) {
-            $rt = ' (RT ' . $item->user->name . ')';
-            $item = $item->retweeted_status;
-        }
-
         $entities = self::formatEntities($item->entities);
-        $tweet = $item->user->name . $rt . ':<br>' . self::replaceEntities($item->full_text, $entities);
+        $tweet = self::replaceEntities($item->full_text, $entities);
 
         return $tweet;
     }
@@ -241,10 +241,6 @@ private function getTweetTitle(stdClass $item) {
     private function getContent(stdClass $item) {
         $result = '';
 
-        if (isset($item->retweeted_status)) {
-            $item = $item->retweeted_status;
-        }
-
         if (isset($item->extended_entities) && isset($item->extended_entities->media) && count($item->extended_entities->media) > 0) {
             foreach ($item->extended_entities->media as $media) {
                 if ($media->type === 'photo') {
@@ -268,20 +264,13 @@ private function getContent(stdClass $item) {
      * @return string
      */
     private function getTweetIcon(stdClass $item) {
-        if (isset($item->retweeted_status)) {
-            $item = $item->retweeted_status;
-        }
-
         return $item->user->profile_image_url_https;
     }
 
     /**
      * @return ?string
      */
     private function getThumbnail(stdClass $item) {
-        if (isset($item->retweeted_status)) {
-            $item = $item->retweeted_status;
-        }
         if (isset($item->entities->media) && $item->entities->media[0]->type === 'photo') {
             return $item->entities->media[0]->media_url_https;
         }