Add endpoints for backfilling history (MSC2716)

scripts-dev/complement.sh

@@ -11,12 +11,10 @@ cd "$(dirname $0)/.."
 docker build -t matrixdotorg/synapse:latest -f docker/Dockerfile .
 
 # Download Complement
-wget -N https://github.com/matrix-org/complement/archive/master.tar.gz
-tar -xzf master.tar.gz
-cd complement-master
+cd ../complement
 
 # Build the Synapse image from Complement, based on the above image we just built
 docker build -t complement-synapse -f dockerfiles/Synapse.Dockerfile ./dockerfiles
 
 # Run the tests on the resulting image!
-COMPLEMENT_BASE_IMAGE=complement-synapse go test -v -count=1 ./tests
+COMPLEMENT_BASE_IMAGE=complement-synapse go test -tags msc2716 -v -count=1 ./tests/main_test.go ./tests/msc2716_test.go

synapse/handlers/message.py

@@ -736,8 +736,12 @@ async def create_and_send_nonmember_event(
                     assert event.internal_metadata.stream_ordering
                     return event, event.internal_metadata.stream_ordering
 
+            prev_events = None
+            if "prev_events" in event_dict:
+                prev_events = event_dict["prev_events"]
+
             event, context = await self.create_event(
-                requester, event_dict, txn_id=txn_id
+                requester, event_dict, txn_id=txn_id, prev_event_ids=prev_events
             )
 
             assert self.hs.is_mine_id(event.sender), "User must be our own: %s" % (

synapse/http/servlet.py

@@ -16,6 +16,7 @@
 """ This module contains base REST classes for constructing REST servlets. """
 
 import logging
+from typing import List
 
 from synapse.api.errors import Codes, SynapseError
 from synapse.util import json_decoder
@@ -147,38 +148,46 @@ def parse_string(
     )
 
 
-def parse_string_from_args(
+def parse_string_value(
+    value,
+    allowed_values,
+    name="",
+    encoding="ascii"
+) -> str:
+    if encoding:
+        try:
+            value = value.decode(encoding)
+        except ValueError:
+            raise SynapseError(
+                400, "Query parameter %r must be %s" % (name, encoding)
+            )
+
+    if allowed_values is not None and value not in allowed_values:
+        message = "Query parameter %r must be one of [%s]" % (
+            name,
+            ", ".join(repr(v) for v in allowed_values),
+        )
+        raise SynapseError(400, message)
+    else:
+        return value
+
+def parse_strings_from_args(
     args,
     name,
     default=None,
     required=False,
     allowed_values=None,
     param_type="string",
     encoding="ascii",
-):
+) -> List[str]:
 
     if not isinstance(name, bytes):
         name = name.encode("ascii")
 
     if name in args:
-        value = args[name][0]
-
-        if encoding:
-            try:
-                value = value.decode(encoding)
-            except ValueError:
-                raise SynapseError(
-                    400, "Query parameter %r must be %s" % (name, encoding)
-                )
-
-        if allowed_values is not None and value not in allowed_values:
-            message = "Query parameter %r must be one of [%s]" % (
-                name,
-                ", ".join(repr(v) for v in allowed_values),
-            )
-            raise SynapseError(400, message)
-        else:
-            return value
+        values = args[name]
+
+        return [parse_string_value(value, allowed_values, name=name, encoding=encoding) for value in values]
     else:
         if required:
             message = "Missing %s query parameter %r" % (param_type, name)
@@ -191,6 +200,31 @@ def parse_string_from_args(
             return default
 
 
+def parse_string_from_args(
+    args,
+    name,
+    default=None,
+    required=False,
+    allowed_values=None,
+    param_type="string",
+    encoding="ascii",
+):
+    strings = parse_strings_from_args(
+        args,
+        name,
+        default=default,
+        required=required,
+        allowed_values=allowed_values,
+        param_type=param_type,
+        encoding=encoding,
+    )
+
+    if isinstance(strings, list) and len(strings):
+        return strings[0]
+
+    # Return the default
+    return strings
+
 def parse_json_value_from_request(request, allow_empty_body=False):
     """Parse a JSON value from the body of a twisted HTTP request.
 
@@ -245,7 +279,6 @@ def parse_json_object_from_request(request, allow_empty_body=False):
 
     return content
 
-
 def assert_params_in_dict(body, required):
     absent = []
     for k in required:

synapse/rest/client/v1/room.py

@@ -37,6 +37,8 @@
     assert_params_in_dict,
     parse_integer,
     parse_json_object_from_request,
+    parse_strings_from_args,
+    parse_integer_from_args,
     parse_string,
 )
 from synapse.logging.opentracing import set_tag
@@ -222,14 +224,26 @@ def register(self, http_server):
     async def on_POST(self, request, room_id, event_type, txn_id=None):
         requester = await self.auth.get_user_by_req(request, allow_guest=True)
         content = parse_json_object_from_request(request)
+        prev_events = parse_strings_from_args(request.args, "prev_event")
+        origin_server_ts = parse_integer(request, "origin_server_ts")
 
         event_dict = {
             "type": event_type,
             "content": content,
             "room_id": room_id,
-            "sender": requester.user.to_string(),
+            "sender": requester.user.to_string()
         }
 
+        if prev_events:
+            event_dict["prev_events"] = prev_events
+
+        # TODO: Add `and requester.app_service`
+        if origin_server_ts:
+            event_dict["origin_server_ts"] = origin_server_ts
+
+        # TODO: I noticed in the Synapse code that we already accept a `ts` query parameter to override
+        # the `origin_server_ts` if the request is coming from an app service.
+        # Do we want to remove in favor of the spec'ed code above
         if b"ts" in request.args and requester.app_service:
             event_dict["origin_server_ts"] = parse_integer(request, "ts", 0)