Merge commit '77b4711bc' into anoa/dinsic_release_1_21_x

* commit '77b4711bc':
  Add cross-signing sigs to the `keys` object (#8234)
  Unread counts fixes (#8254)
  Fix a regression from calling read_templates. (#8252)

changelog.d/8234.misc

@@ -0,0 +1 @@
+Refactor queries for device keys and cross-signatures.

changelog.d/8252.feature

@@ -0,0 +1 @@
+Use the default template file when its equivalent is not found in a custom template directory.

changelog.d/8254.feature

@@ -0,0 +1 @@
+Add unread messages count to sync responses, as specified in [MSC2654](https://github.com/matrix-org/matrix-doc/pull/2654).

synapse/config/saml2_config.py

@@ -171,7 +171,7 @@ def read_config(self, config, **kwargs):
 
         self.saml2_error_html_template = self.read_templates(
             ["saml_error.html"], saml2_config.get("template_dir")
-        )
+        )[0]
 
     def _default_saml_config_dict(
         self, required_attributes: set, optional_attributes: set

synapse/push/push_tools.py

@@ -36,7 +36,7 @@ async def get_badge_count(store, user_id):
             )
             # return one badge count per conversation, as count per
             # message is so noisy as to be almost useless
-            badge += 1 if notifs["unread_count"] else 0
+            badge += 1 if notifs["notify_count"] else 0
     return badge
 
 

synapse/storage/databases/main/devices.py

@@ -291,15 +291,9 @@ async def _get_device_update_edus_by_remote(
                 prev_id = stream_id
 
                 if device is not None:
-                    key_json = device.key_json
-                    if key_json:
-                        result["keys"] = db_to_json(key_json)
-
-                        if device.signatures:
-                            for sig_user_id, sigs in device.signatures.items():
-                                result["keys"].setdefault("signatures", {}).setdefault(
-                                    sig_user_id, {}
-                                ).update(sigs)
+                    keys = device.keys
+                    if keys:
+                        result["keys"] = keys
 
                     device_display_name = device.display_name
                     if device_display_name:

synapse/storage/databases/main/end_to_end_keys.py

@@ -43,12 +43,8 @@ class DeviceKeyLookupResult:
 
     # the key data from e2e_device_keys_json. Typically includes fields like
     # "algorithm", "keys" (including the curve25519 identity key and the ed25519 signing
-    # key) and "signatures" (a signature of the structure by the ed25519 key)
-    key_json = attr.ib(type=Optional[str])
-
-    # cross-signing sigs on this device.
-    # dict from (signing user_id)->(signing device_id)->sig
-    signatures = attr.ib(type=Optional[Dict[str, Dict[str, str]]], factory=dict)
+    # key) and "signatures" (a map from (user id) to (key id/device_id) to signature.)
+    keys = attr.ib(type=Optional[JsonDict])
 
 
 class EndToEndKeyWorkerStore(SQLBaseStore):
@@ -70,15 +66,9 @@ async def get_e2e_device_keys_for_federation_query(
             for device_id, device in user_devices.items():
                 result = {"device_id": device_id}
 
-                key_json = device.key_json
-                if key_json:
-                    result["keys"] = db_to_json(key_json)
-
-                    if device.signatures:
-                        for sig_user_id, sigs in device.signatures.items():
-                            result["keys"].setdefault("signatures", {}).setdefault(
-                                sig_user_id, {}
-                            ).update(sigs)
+                keys = device.keys
+                if keys:
+                    result["keys"] = keys
 
                 device_display_name = device.display_name
                 if device_display_name:
@@ -114,16 +104,11 @@ async def get_e2e_device_keys_for_cs_api(
         for user_id, device_keys in results.items():
             rv[user_id] = {}
             for device_id, device_info in device_keys.items():
-                r = db_to_json(device_info.key_json)
+                r = device_info.keys
                 r["unsigned"] = {}
                 display_name = device_info.display_name
                 if display_name is not None:
                     r["unsigned"]["device_display_name"] = display_name
-                if device_info.signatures:
-                    for sig_user_id, sigs in device_info.signatures.items():
-                        r.setdefault("signatures", {}).setdefault(
-                            sig_user_id, {}
-                        ).update(sigs)
                 rv[user_id][device_id] = r
 
         return rv
@@ -140,6 +125,9 @@ async def get_e2e_device_keys_and_signatures(
         Any cross-signatures made on the keys by the owner of the device are also
         included.
 
+        The cross-signatures are added to the `signatures` field within the `keys`
+        object in the response.
+
         Args:
             query_list: List of pairs of user_ids and device_ids. Device id can be None
                 to indicate "all devices for this user"
@@ -170,7 +158,7 @@ async def get_e2e_device_keys_and_signatures(
             (user_id, device_id)
             for user_id, dev in result.items()
             for device_id, d in dev.items()
-            if d is not None
+            if d is not None and d.keys is not None
         )
 
         for batch in batch_iter(signature_query, 50):
@@ -183,8 +171,9 @@ async def get_e2e_device_keys_and_signatures(
             # add each cross-signing signature to the correct device in the result dict.
             for (user_id, key_id, device_id, signature) in cross_sigs_result:
                 target_device_result = result[user_id][device_id]
-                target_device_signatures = target_device_result.signatures
-
+                target_device_signatures = target_device_result.keys.setdefault(
+                    "signatures", {}
+                )
                 signing_user_signatures = target_device_signatures.setdefault(
                     user_id, {}
                 )
@@ -240,7 +229,7 @@ def _get_e2e_device_keys_txn(
             if include_deleted_devices:
                 deleted_devices.remove((user_id, device_id))
             result.setdefault(user_id, {})[device_id] = DeviceKeyLookupResult(
-                display_name, key_json
+                display_name, db_to_json(key_json) if key_json else None
             )
 
         if include_deleted_devices:

synapse/storage/databases/main/schema/delta/58/15unread_count.sql

@@ -19,8 +19,8 @@
 
 -- Add columns to event_push_actions and event_push_actions_staging to track unread
 -- messages and calculate unread counts.
-ALTER TABLE event_push_actions_staging ADD COLUMN unread SMALLINT NOT NULL DEFAULT 0;
-ALTER TABLE event_push_actions ADD COLUMN unread SMALLINT NOT NULL DEFAULT 0;
+ALTER TABLE event_push_actions_staging ADD COLUMN unread SMALLINT;
+ALTER TABLE event_push_actions ADD COLUMN unread SMALLINT;
 
 -- Add column to event_push_summary
-ALTER TABLE event_push_summary ADD COLUMN unread_count BIGINT NOT NULL DEFAULT 0;
+ALTER TABLE event_push_summary ADD COLUMN unread_count BIGINT;