From d99b9acfd9e0431edc8c98fe448221b2f0c7b822 Mon Sep 17 00:00:00 2001
From: Mathieu Velten <mathieuv@matrix.org>
Date: Fri, 16 Sep 2022 16:39:15 +0200
Subject: [PATCH 01/20] Add tests for typing/presence/to_device EDUs during
 fast joins

Signed-off-by: Mathieu Velten <mathieuv@matrix.org>
---
 ...federation_room_join_partial_state_test.go | 138 +++++++++++++++++-
 1 file changed, 137 insertions(+), 1 deletion(-)

diff --git a/tests/federation_room_join_partial_state_test.go b/tests/federation_room_join_partial_state_test.go
index 279cde5f..791788a9 100644
--- a/tests/federation_room_join_partial_state_test.go
+++ b/tests/federation_room_join_partial_state_test.go
@@ -188,6 +188,142 @@ func TestPartialStateJoin(t *testing.T) {
 		t.Logf("Alice sent event event ID %s", eventID)
 	})
 
+	// we should be able to receive typing EDU over federation during the resync
+	t.Run("CanReceiveTypingDuringPartialStateJoin", func(t *testing.T) {
+		deployment := Deploy(t, b.BlueprintAlice)
+		defer deployment.Destroy(t)
+		alice := deployment.Client(t, "hs1", "@alice:hs1")
+
+		server := createTestServer(t, deployment)
+		cancel := server.Listen()
+		defer cancel()
+		serverRoom := createTestRoom(t, server, alice.GetDefaultRoomVersion(t))
+		psjResult := beginPartialStateJoin(t, server, serverRoom, alice)
+		defer psjResult.Destroy()
+
+		derekUserId := psjResult.Server.UserID("derek")
+
+		content, _ := json.Marshal(map[string]interface{}{
+			"room_id": serverRoom.RoomID,
+			"user_id": derekUserId,
+			"typing":  true,
+		})
+		edu := gomatrixserverlib.EDU{
+			Type:    "m.typing",
+			Content: content,
+		}
+		psjResult.Server.MustSendTransaction(t, deployment, "hs1", []json.RawMessage{}, []gomatrixserverlib.EDU{edu})
+
+		psjResult.FinishStateRequest()
+		alice.MustSyncUntil(t,
+			client.SyncReq{},
+			client.SyncEphemeralHas(serverRoom.RoomID, func(result gjson.Result) bool {
+				if result.Get("type").Str != "m.typing" {
+					return false
+				}
+				for _, item := range result.Get("content").Get("user_ids").Array() {
+					if item.Str == derekUserId {
+						return true
+					}
+				}
+				return false
+			}),
+		)
+	})
+
+	// we should be able to receive presence EDU over federation during the resync
+	t.Run("CanReceivePresenceDuringPartialStateJoin", func(t *testing.T) {
+		deployment := Deploy(t, b.BlueprintAlice)
+		defer deployment.Destroy(t)
+		alice := deployment.Client(t, "hs1", "@alice:hs1")
+
+		server := createTestServer(t, deployment)
+		cancel := server.Listen()
+		defer cancel()
+		serverRoom := createTestRoom(t, server, alice.GetDefaultRoomVersion(t))
+		psjResult := beginPartialStateJoin(t, server, serverRoom, alice)
+		defer psjResult.Destroy()
+
+		derekUserId := psjResult.Server.UserID("derek")
+
+		content, _ := json.Marshal(map[string]interface{}{
+			"push": []map[string]interface{}{
+				map[string]interface{}{
+					"user_id":         derekUserId,
+					"presence":        "online",
+					"last_active_ago": 100,
+				},
+			},
+		})
+		edu := gomatrixserverlib.EDU{
+			Type:    "m.presence",
+			Content: content,
+		}
+		psjResult.Server.MustSendTransaction(t, deployment, "hs1", []json.RawMessage{}, []gomatrixserverlib.EDU{edu})
+
+		// for now we just check that the fed transaction is accepted,
+		// presence is not always going down the sync for some reason
+
+		// psjResult.FinishStateRequest()
+		// alice.MustSyncUntil(t,
+		// 	client.SyncReq{},
+		// 	func(userID string, sync gjson.Result) error {
+		// 		for _, e := range sync.Get("presence").Get("events").Array() {
+		// 			t.Logf("lllll %s", e)
+		// 			if e.Get("sender").Str == derekUserId {
+		// 				return nil
+		// 			}
+		// 		}
+		// 		return fmt.Errorf("No presence update from %s", derekUserId)
+		// 	},
+		// )
+	})
+
+	// we should be able to receive to_device EDU over federation during the resync
+	t.Run("CanReceiveToDeviceDuringPartialStateJoin", func(t *testing.T) {
+		deployment := Deploy(t, b.BlueprintAlice)
+		defer deployment.Destroy(t)
+		alice := deployment.Client(t, "hs1", "@alice:hs1")
+
+		server := createTestServer(t, deployment)
+		cancel := server.Listen()
+		defer cancel()
+		serverRoom := createTestRoom(t, server, alice.GetDefaultRoomVersion(t))
+		psjResult := beginPartialStateJoin(t, server, serverRoom, alice)
+		defer psjResult.Destroy()
+
+		derekUserId := psjResult.Server.UserID("derek")
+
+		content, _ := json.Marshal(map[string]interface{}{
+			"message_id": "hiezohf6Hoo7kaev",
+			"sender":     derekUserId,
+			"type":       "m.test",
+			"messages": map[string]interface{}{
+				alice.UserID: map[string]interface{}{
+					"*": map[string]interface{}{},
+				},
+			},
+		})
+		edu := gomatrixserverlib.EDU{
+			Type:    "m.direct_to_device",
+			Content: content,
+		}
+		psjResult.Server.MustSendTransaction(t, deployment, "hs1", []json.RawMessage{}, []gomatrixserverlib.EDU{edu})
+
+		psjResult.FinishStateRequest()
+		alice.MustSyncUntil(t,
+			client.SyncReq{},
+			func(userID string, sync gjson.Result) error {
+				for _, e := range sync.Get("to_device").Get("events").Array() {
+					if e.Get("sender").Str == derekUserId && e.Get("type").Str == "m.test" {
+						return nil
+					}
+				}
+				return fmt.Errorf("No to_device update from %s", derekUserId)
+			},
+		)
+	})
+
 	// we should be able to receive events over federation during the resync
 	t.Run("CanReceiveEventsDuringPartialStateJoin", func(t *testing.T) {
 		deployment := Deploy(t, b.BlueprintAlice)
@@ -1575,7 +1711,7 @@ func testReceiveEventDuringPartialStateJoin(
 		"GET",
 		[]string{"_matrix", "client", "v3", "rooms", psjResult.ServerRoom.RoomID, "state", "m.room.member", "@non-existent:remote"},
 	)
-	
+
 	// check the server's idea of the state at the event. We do this by making a `state_ids` request over federation
 	stateReq = gomatrixserverlib.NewFederationRequest("GET", "hs1",
 		fmt.Sprintf("/_matrix/federation/v1/state_ids/%s?event_id=%s",

From 7778cca85aa7c92422e8633c088d67bbdbfa7083 Mon Sep 17 00:00:00 2001
From: Mathieu Velten <mathieuv@matrix.org>
Date: Mon, 19 Sep 2022 15:22:53 +0200
Subject: [PATCH 02/20] Add receipt test

---
 ...federation_room_join_partial_state_test.go | 49 +++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/tests/federation_room_join_partial_state_test.go b/tests/federation_room_join_partial_state_test.go
index 791788a9..b6237409 100644
--- a/tests/federation_room_join_partial_state_test.go
+++ b/tests/federation_room_join_partial_state_test.go
@@ -324,6 +324,55 @@ func TestPartialStateJoin(t *testing.T) {
 		)
 	})
 
+	// we should be able to receipt typing EDU over federation during the resync
+	t.Run("CanReceiveReceiptDuringPartialStateJoin", func(t *testing.T) {
+		deployment := Deploy(t, b.BlueprintAlice)
+		defer deployment.Destroy(t)
+		alice := deployment.Client(t, "hs1", "@alice:hs1")
+
+		server := createTestServer(t, deployment)
+		cancel := server.Listen()
+		defer cancel()
+		serverRoom := createTestRoom(t, server, alice.GetDefaultRoomVersion(t))
+		psjResult := beginPartialStateJoin(t, server, serverRoom, alice)
+		defer psjResult.Destroy()
+
+		derekUserId := psjResult.Server.UserID("derek")
+
+		content, _ := json.Marshal(map[string]interface{}{
+			serverRoom.RoomID: map[string]interface{}{
+				"m.read": map[string]interface{}{
+					derekUserId: map[string]interface{}{
+						"data": map[string]interface{}{
+							"ts": 1436451550453,
+						},
+						"event_ids": []string{"mytesteventid"},
+					},
+				},
+			},
+		})
+		edu := gomatrixserverlib.EDU{
+			Type:    "m.receipt",
+			Content: content,
+		}
+		psjResult.Server.MustSendTransaction(t, deployment, "hs1", []json.RawMessage{}, []gomatrixserverlib.EDU{edu})
+
+		psjResult.FinishStateRequest()
+		alice.MustSyncUntil(t,
+			client.SyncReq{},
+			client.SyncEphemeralHas(serverRoom.RoomID, func(result gjson.Result) bool {
+				if result.Get("type").Str != "m.receipt" {
+					return false
+				}
+
+				if result.Get("content").Get("mytesteventid").Get("m\\.read").Get(strings.Replace(derekUserId, ".", "\\.", -1)).Get("ts").Int() == 1436451550453 {
+					return true
+				}
+				return false
+			}),
+		)
+	})
+
 	// we should be able to receive events over federation during the resync
 	t.Run("CanReceiveEventsDuringPartialStateJoin", func(t *testing.T) {
 		deployment := Deploy(t, b.BlueprintAlice)

From 0300eb79d35c3fa922f81bb3c195f89238943f23 Mon Sep 17 00:00:00 2001
From: Mathieu Velten <mathieuv@matrix.org>
Date: Mon, 19 Sep 2022 15:59:57 +0200
Subject: [PATCH 03/20] Add device list and signing key update EDU

---
 ...federation_room_join_partial_state_test.go | 60 ++++++++++++++++++-
 1 file changed, 59 insertions(+), 1 deletion(-)

diff --git a/tests/federation_room_join_partial_state_test.go b/tests/federation_room_join_partial_state_test.go
index b6237409..e8224afc 100644
--- a/tests/federation_room_join_partial_state_test.go
+++ b/tests/federation_room_join_partial_state_test.go
@@ -324,7 +324,7 @@ func TestPartialStateJoin(t *testing.T) {
 		)
 	})
 
-	// we should be able to receipt typing EDU over federation during the resync
+	// we should be able to receive receipt EDU over federation during the resync
 	t.Run("CanReceiveReceiptDuringPartialStateJoin", func(t *testing.T) {
 		deployment := Deploy(t, b.BlueprintAlice)
 		defer deployment.Destroy(t)
@@ -373,6 +373,64 @@ func TestPartialStateJoin(t *testing.T) {
 		)
 	})
 
+	// we should be able to receive device list update EDU over federation during the resync
+	t.Run("CanReceiveDeviceListUpdateDuringPartialStateJoin", func(t *testing.T) {
+		deployment := Deploy(t, b.BlueprintAlice)
+		defer deployment.Destroy(t)
+		alice := deployment.Client(t, "hs1", "@alice:hs1")
+
+		server := createTestServer(t, deployment)
+		cancel := server.Listen()
+		defer cancel()
+		serverRoom := createTestRoom(t, server, alice.GetDefaultRoomVersion(t))
+		psjResult := beginPartialStateJoin(t, server, serverRoom, alice)
+		defer psjResult.Destroy()
+
+		derekUserId := psjResult.Server.UserID("derek")
+
+		content, _ := json.Marshal(map[string]interface{}{
+			"device_id": "QBUAZIFURK",
+			"stream_id": 1,
+			"user_id":   derekUserId,
+		})
+		edu := gomatrixserverlib.EDU{
+			Type:    "m.device_list_update",
+			Content: content,
+		}
+		psjResult.Server.MustSendTransaction(t, deployment, "hs1", []json.RawMessage{}, []gomatrixserverlib.EDU{edu})
+
+		// If we want to check the sync we need to have an encrypted room,
+		// for now just check that the fed transaction is accepted.
+	})
+
+	// we should be able to receive signing key update EDU over federation during the resync
+	t.Run("CanReceiveSigningKeyUpdateDuringPartialStateJoin", func(t *testing.T) {
+		deployment := Deploy(t, b.BlueprintAlice)
+		defer deployment.Destroy(t)
+		alice := deployment.Client(t, "hs1", "@alice:hs1")
+
+		server := createTestServer(t, deployment)
+		cancel := server.Listen()
+		defer cancel()
+		serverRoom := createTestRoom(t, server, alice.GetDefaultRoomVersion(t))
+		psjResult := beginPartialStateJoin(t, server, serverRoom, alice)
+		defer psjResult.Destroy()
+
+		derekUserId := psjResult.Server.UserID("derek")
+
+		content, _ := json.Marshal(map[string]interface{}{
+			"user_id": derekUserId,
+		})
+		edu := gomatrixserverlib.EDU{
+			Type:    "m.signing_key_update",
+			Content: content,
+		}
+		psjResult.Server.MustSendTransaction(t, deployment, "hs1", []json.RawMessage{}, []gomatrixserverlib.EDU{edu})
+
+		// If we want to check the sync we need to have an encrypted room,
+		// for now just check that the fed transaction is accepted.
+	})
+
 	// we should be able to receive events over federation during the resync
 	t.Run("CanReceiveEventsDuringPartialStateJoin", func(t *testing.T) {
 		deployment := Deploy(t, b.BlueprintAlice)

From 7331ebc6d08f3328dcab7079432472491359f995 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 11 Oct 2022 14:19:18 +0100
Subject: [PATCH 04/20] Ensure we detect matrix-org/synapse#13684

---
 tests/federation_room_join_partial_state_test.go | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tests/federation_room_join_partial_state_test.go b/tests/federation_room_join_partial_state_test.go
index e8224afc..2bd2a249 100644
--- a/tests/federation_room_join_partial_state_test.go
+++ b/tests/federation_room_join_partial_state_test.go
@@ -214,6 +214,14 @@ func TestPartialStateJoin(t *testing.T) {
 		}
 		psjResult.Server.MustSendTransaction(t, deployment, "hs1", []json.RawMessage{}, []gomatrixserverlib.EDU{edu})
 
+		// Alice should still be able to see incoming PDUs in the room during
+		// the resync; the earlier EDU shouldn't interfere with this.
+		// (See https://github.com/matrix-org/synapse/issues/13684)
+		event := psjResult.CreateMessageEvent(t, "charlie", nil)
+		serverRoom.AddEvent(event)
+		server.MustSendTransaction(t, deployment, "hs1", []json.RawMessage{event.JSON()}, nil)
+		awaitEventViaSync(t, alice, serverRoom.RoomID, event.EventID(), "")
+
 		psjResult.FinishStateRequest()
 		alice.MustSyncUntil(t,
 			client.SyncReq{},

From 72421b4b2ada42ebe0887bd98bb531d50b0bc7bc Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 11 Oct 2022 14:47:24 +0100
Subject: [PATCH 05/20] Test that Alice sees typing before & after resync

---
 ...federation_room_join_partial_state_test.go | 46 ++++++++++++++-----
 1 file changed, 35 insertions(+), 11 deletions(-)

diff --git a/tests/federation_room_join_partial_state_test.go b/tests/federation_room_join_partial_state_test.go
index 2bd2a249..528da9bb 100644
--- a/tests/federation_room_join_partial_state_test.go
+++ b/tests/federation_room_join_partial_state_test.go
@@ -201,8 +201,8 @@ func TestPartialStateJoin(t *testing.T) {
 		psjResult := beginPartialStateJoin(t, server, serverRoom, alice)
 		defer psjResult.Destroy()
 
+		// Derek starts typing in the room.
 		derekUserId := psjResult.Server.UserID("derek")
-
 		content, _ := json.Marshal(map[string]interface{}{
 			"room_id": serverRoom.RoomID,
 			"user_id": derekUserId,
@@ -214,6 +214,21 @@ func TestPartialStateJoin(t *testing.T) {
 		}
 		psjResult.Server.MustSendTransaction(t, deployment, "hs1", []json.RawMessage{}, []gomatrixserverlib.EDU{edu})
 
+		// Alice should be able to see that Derek is typing (even though HS1 is resyncing).
+		alice.MustSyncUntil(t,
+			client.SyncReq{},
+			client.SyncEphemeralHas(serverRoom.RoomID, func(result gjson.Result) bool {
+				if result.Get("type").Str != "m.typing" {
+					return false
+				}
+				user_ids := result.Get("content.user_ids").Array()
+				if len(user_ids) != 1 {
+					return false
+				}
+				return user_ids[0].Str == derekUserID
+			}),
+		)
+
 		// Alice should still be able to see incoming PDUs in the room during
 		// the resync; the earlier EDU shouldn't interfere with this.
 		// (See https://github.com/matrix-org/synapse/issues/13684)
@@ -222,21 +237,30 @@ func TestPartialStateJoin(t *testing.T) {
 		server.MustSendTransaction(t, deployment, "hs1", []json.RawMessage{event.JSON()}, nil)
 		awaitEventViaSync(t, alice, serverRoom.RoomID, event.EventID(), "")
 
+		// The resync completes.
 		psjResult.FinishStateRequest()
+
+		// Derek stops typing.
+		content, _ = json.Marshal(map[string]interface{}{
+			"room_id": serverRoom.RoomID,
+			"user_id": derekUserId,
+			"typing":  false,
+		})
+		edu := gomatrixserverlib.EDU{
+			Type:    "m.typing",
+			Content: content,
+		}
+		psjResult.Server.MustSendTransaction(t, deployment, "hs1", []json.RawMessage{}, []gomatrixserverlib.EDU{edu})
+
+		// Alice should be able to see that no-one is typing.
 		alice.MustSyncUntil(t,
 			client.SyncReq{},
 			client.SyncEphemeralHas(serverRoom.RoomID, func(result gjson.Result) bool {
-				if result.Get("type").Str != "m.typing" {
-					return false
-				}
-				for _, item := range result.Get("content").Get("user_ids").Array() {
-					if item.Str == derekUserId {
-						return true
-					}
-				}
-				return false
+				return result.Get("type").Str == "m.typing" &&
+					result.Get("content.user_ids.#").Int() == 0
 			}),
 		)
+
 	})
 
 	// we should be able to receive presence EDU over federation during the resync
@@ -269,7 +293,7 @@ func TestPartialStateJoin(t *testing.T) {
 		}
 		psjResult.Server.MustSendTransaction(t, deployment, "hs1", []json.RawMessage{}, []gomatrixserverlib.EDU{edu})
 
-		// for now we just check that the fed transaction is accepted,
+		// TODO: for now we just check that the fed transaction is accepted,
 		// presence is not always going down the sync for some reason
 
 		// psjResult.FinishStateRequest()

From 778bf65d82aac166ca196d3b42f8610108b0100e Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 11 Oct 2022 15:56:49 +0100
Subject: [PATCH 06/20] We should test presence EDUs during resync

---
 tests/federation_room_join_partial_state_test.go | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tests/federation_room_join_partial_state_test.go b/tests/federation_room_join_partial_state_test.go
index 528da9bb..52f54cc5 100644
--- a/tests/federation_room_join_partial_state_test.go
+++ b/tests/federation_room_join_partial_state_test.go
@@ -294,9 +294,10 @@ func TestPartialStateJoin(t *testing.T) {
 		psjResult.Server.MustSendTransaction(t, deployment, "hs1", []json.RawMessage{}, []gomatrixserverlib.EDU{edu})
 
 		// TODO: for now we just check that the fed transaction is accepted,
-		// presence is not always going down the sync for some reason
-
-		// psjResult.FinishStateRequest()
+		// presence is not always going down the sync for some reason.
+		// Unclear if this is an existing Synapse bug or a new problem specific
+		// to partial joins.
+		//
 		// alice.MustSyncUntil(t,
 		// 	client.SyncReq{},
 		// 	func(userID string, sync gjson.Result) error {
@@ -309,6 +310,8 @@ func TestPartialStateJoin(t *testing.T) {
 		// 		return fmt.Errorf("No presence update from %s", derekUserId)
 		// 	},
 		// )
+
+		psjResult.FinishStateRequest()
 	})
 
 	// we should be able to receive to_device EDU over federation during the resync

From 40978ec0d6d53f70550401874cb6222c4a32b679 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 11 Oct 2022 15:57:15 +0100
Subject: [PATCH 07/20] Test to_device during resync & look for message_id

---
 tests/federation_room_join_partial_state_test.go | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/tests/federation_room_join_partial_state_test.go b/tests/federation_room_join_partial_state_test.go
index 52f54cc5..18bcb3fd 100644
--- a/tests/federation_room_join_partial_state_test.go
+++ b/tests/federation_room_join_partial_state_test.go
@@ -327,10 +327,11 @@ func TestPartialStateJoin(t *testing.T) {
 		psjResult := beginPartialStateJoin(t, server, serverRoom, alice)
 		defer psjResult.Destroy()
 
+		// Send a to-device message from Derek to Alice.
 		derekUserId := psjResult.Server.UserID("derek")
-
+		messageId := "hiezohf6Hoo7kaev"
 		content, _ := json.Marshal(map[string]interface{}{
-			"message_id": "hiezohf6Hoo7kaev",
+			"message_id": messageId,
 			"sender":     derekUserId,
 			"type":       "m.test",
 			"messages": map[string]interface{}{
@@ -345,18 +346,21 @@ func TestPartialStateJoin(t *testing.T) {
 		}
 		psjResult.Server.MustSendTransaction(t, deployment, "hs1", []json.RawMessage{}, []gomatrixserverlib.EDU{edu})
 
-		psjResult.FinishStateRequest()
+		// Alice should see Derek's to-device message when she syncs.
 		alice.MustSyncUntil(t,
 			client.SyncReq{},
 			func(userID string, sync gjson.Result) error {
-				for _, e := range sync.Get("to_device").Get("events").Array() {
-					if e.Get("sender").Str == derekUserId && e.Get("type").Str == "m.test" {
+				for _, e := range sync.Get("to_device.events").Array() {
+					if e.Get("sender").Str == derekUserId &&
+						e.Get("type").Str == "m.test" &&
+						e.Get("message_id").Str == messageId {
 						return nil
 					}
 				}
 				return fmt.Errorf("No to_device update from %s", derekUserId)
 			},
 		)
+		psjResult.FinishStateRequest()
 	})
 
 	// we should be able to receive receipt EDU over federation during the resync

From a9a19558c36cb33fba328e2f5564f1a4620f2416 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 11 Oct 2022 16:08:44 +0100
Subject: [PATCH 08/20] Check read receipts are visible during resync

---
 tests/federation_room_join_partial_state_test.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/federation_room_join_partial_state_test.go b/tests/federation_room_join_partial_state_test.go
index 18bcb3fd..c3212452 100644
--- a/tests/federation_room_join_partial_state_test.go
+++ b/tests/federation_room_join_partial_state_test.go
@@ -378,6 +378,7 @@ func TestPartialStateJoin(t *testing.T) {
 
 		derekUserId := psjResult.Server.UserID("derek")
 
+		// Derek sends a read receipt into the room.
 		content, _ := json.Marshal(map[string]interface{}{
 			serverRoom.RoomID: map[string]interface{}{
 				"m.read": map[string]interface{}{
@@ -396,7 +397,7 @@ func TestPartialStateJoin(t *testing.T) {
 		}
 		psjResult.Server.MustSendTransaction(t, deployment, "hs1", []json.RawMessage{}, []gomatrixserverlib.EDU{edu})
 
-		psjResult.FinishStateRequest()
+		// Alice should be able to see Derek's read receipt during the resync
 		alice.MustSyncUntil(t,
 			client.SyncReq{},
 			client.SyncEphemeralHas(serverRoom.RoomID, func(result gjson.Result) bool {
@@ -410,6 +411,7 @@ func TestPartialStateJoin(t *testing.T) {
 				return false
 			}),
 		)
+		psjResult.FinishStateRequest()
 	})
 
 	// we should be able to receive device list update EDU over federation during the resync

From 655998c9d96c2de63fe4eba2c5477620432804bc Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 11 Oct 2022 17:00:54 +0100
Subject: [PATCH 09/20] Try to appease the go grammar idk

---
 tests/federation_room_join_partial_state_test.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/federation_room_join_partial_state_test.go b/tests/federation_room_join_partial_state_test.go
index ccb04293..b552855b 100644
--- a/tests/federation_room_join_partial_state_test.go
+++ b/tests/federation_room_join_partial_state_test.go
@@ -309,8 +309,8 @@ func TestPartialStateJoin(t *testing.T) {
 		alice.MustSyncUntil(t,
 			client.SyncReq{},
 			client.SyncEphemeralHas(serverRoom.RoomID, func(result gjson.Result) bool {
-				return result.Get("type").Str == "m.typing" &&
-					result.Get("content.user_ids.#").Int() == 0
+				return (result.Get("type").Str == "m.typing" &&
+					result.Get("content.user_ids.#").Int() == 0)
 			}),
 		)
 

From 295e973a0ad792e03ce8b7343cbebdf6d95effd6 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 11 Oct 2022 17:36:54 +0100
Subject: [PATCH 10/20] Fixups, IDK why my VSCode installation is borked

---
 tests/federation_room_join_partial_state_test.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/federation_room_join_partial_state_test.go b/tests/federation_room_join_partial_state_test.go
index b552855b..bb3fd853 100644
--- a/tests/federation_room_join_partial_state_test.go
+++ b/tests/federation_room_join_partial_state_test.go
@@ -278,7 +278,7 @@ func TestPartialStateJoin(t *testing.T) {
 				if len(user_ids) != 1 {
 					return false
 				}
-				return user_ids[0].Str == derekUserID
+				return user_ids[0].Str == derekUserId
 			}),
 		)
 
@@ -299,7 +299,7 @@ func TestPartialStateJoin(t *testing.T) {
 			"user_id": derekUserId,
 			"typing":  false,
 		})
-		edu := gomatrixserverlib.EDU{
+		edu = gomatrixserverlib.EDU{
 			Type:    "m.typing",
 			Content: content,
 		}

From 97dfc66ecbeb22c3ea52f9f4a0d7b57fa8b1f414 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Fri, 21 Oct 2022 17:45:49 +0100
Subject: [PATCH 11/20] Mark known failures as skips

---
 ...federation_room_join_partial_state_test.go | 32 +++++++++----------
 1 file changed, 15 insertions(+), 17 deletions(-)

diff --git a/tests/federation_room_join_partial_state_test.go b/tests/federation_room_join_partial_state_test.go
index bb3fd853..77fc22a1 100644
--- a/tests/federation_room_join_partial_state_test.go
+++ b/tests/federation_room_join_partial_state_test.go
@@ -220,6 +220,7 @@ func TestPartialStateJoin(t *testing.T) {
 
 	// we should be able to send events in the room, during the resync
 	t.Run("CanSendEventsDuringPartialStateJoin", func(t *testing.T) {
+		// See https://github.com/matrix-org/synapse/issues/12997")
 		t.Skip("Cannot yet send events during resync")
 		alice := deployment.RegisterUser(t, "hs1", "t3alice", "secret", false)
 
@@ -318,6 +319,8 @@ func TestPartialStateJoin(t *testing.T) {
 
 	// we should be able to receive presence EDU over federation during the resync
 	t.Run("CanReceivePresenceDuringPartialStateJoin", func(t *testing.T) {
+		// See https://github.com/matrix-org/synapse/issues/13008")
+		t.Skip("Presence EDUs are currently dropped during a resync")
 		deployment := Deploy(t, b.BlueprintAlice)
 		defer deployment.Destroy(t)
 		alice := deployment.Client(t, "hs1", "@alice:hs1")
@@ -346,23 +349,18 @@ func TestPartialStateJoin(t *testing.T) {
 		}
 		psjResult.Server.MustSendTransaction(t, deployment, "hs1", []json.RawMessage{}, []gomatrixserverlib.EDU{edu})
 
-		// TODO: for now we just check that the fed transaction is accepted,
-		// presence is not always going down the sync for some reason.
-		// Unclear if this is an existing Synapse bug or a new problem specific
-		// to partial joins.
-		//
-		// alice.MustSyncUntil(t,
-		// 	client.SyncReq{},
-		// 	func(userID string, sync gjson.Result) error {
-		// 		for _, e := range sync.Get("presence").Get("events").Array() {
-		// 			t.Logf("lllll %s", e)
-		// 			if e.Get("sender").Str == derekUserId {
-		// 				return nil
-		// 			}
-		// 		}
-		// 		return fmt.Errorf("No presence update from %s", derekUserId)
-		// 	},
-		// )
+		alice.MustSyncUntil(t,
+			client.SyncReq{},
+			func(userID string, sync gjson.Result) error {
+				for _, e := range sync.Get("presence").Get("events").Array() {
+					t.Logf("lllll %s", e)
+					if e.Get("sender").Str == derekUserId {
+						return nil
+					}
+				}
+				return fmt.Errorf("No presence update from %s", derekUserId)
+			},
+		)
 
 		psjResult.FinishStateRequest()
 	})

From f654e595e97503022d7201f0a8cb33147676cefd Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Fri, 21 Oct 2022 18:05:17 +0100
Subject: [PATCH 12/20] Lazy-load-sync during receipt test

---
 tests/federation_room_join_partial_state_test.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/federation_room_join_partial_state_test.go b/tests/federation_room_join_partial_state_test.go
index 77fc22a1..e08e71c2 100644
--- a/tests/federation_room_join_partial_state_test.go
+++ b/tests/federation_room_join_partial_state_test.go
@@ -450,7 +450,9 @@ func TestPartialStateJoin(t *testing.T) {
 
 		// Alice should be able to see Derek's read receipt during the resync
 		alice.MustSyncUntil(t,
-			client.SyncReq{},
+			client.SyncReq{
+				Filter: buildLazyLoadingSyncFilter(nil),
+			},
 			client.SyncEphemeralHas(serverRoom.RoomID, func(result gjson.Result) bool {
 				if result.Get("type").Str != "m.receipt" {
 					return false

From c7ae75de47a94388c622942c8b3553a894339b26 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Fri, 21 Oct 2022 18:17:18 +0100
Subject: [PATCH 13/20] okay, you have to lazy-load-sync during a resync

---
 tests/federation_room_join_partial_state_test.go | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/tests/federation_room_join_partial_state_test.go b/tests/federation_room_join_partial_state_test.go
index e08e71c2..740c625b 100644
--- a/tests/federation_room_join_partial_state_test.go
+++ b/tests/federation_room_join_partial_state_test.go
@@ -270,7 +270,9 @@ func TestPartialStateJoin(t *testing.T) {
 
 		// Alice should be able to see that Derek is typing (even though HS1 is resyncing).
 		alice.MustSyncUntil(t,
-			client.SyncReq{},
+			client.SyncReq{
+				Filter: buildLazyLoadingSyncFilter(nil),
+			},
 			client.SyncEphemeralHas(serverRoom.RoomID, func(result gjson.Result) bool {
 				if result.Get("type").Str != "m.typing" {
 					return false
@@ -308,7 +310,9 @@ func TestPartialStateJoin(t *testing.T) {
 
 		// Alice should be able to see that no-one is typing.
 		alice.MustSyncUntil(t,
-			client.SyncReq{},
+			client.SyncReq{
+				Filter: buildLazyLoadingSyncFilter(nil),
+			},
 			client.SyncEphemeralHas(serverRoom.RoomID, func(result gjson.Result) bool {
 				return (result.Get("type").Str == "m.typing" &&
 					result.Get("content.user_ids.#").Int() == 0)
@@ -350,7 +354,9 @@ func TestPartialStateJoin(t *testing.T) {
 		psjResult.Server.MustSendTransaction(t, deployment, "hs1", []json.RawMessage{}, []gomatrixserverlib.EDU{edu})
 
 		alice.MustSyncUntil(t,
-			client.SyncReq{},
+			client.SyncReq{
+				Filter: buildLazyLoadingSyncFilter(nil),
+			},
 			func(userID string, sync gjson.Result) error {
 				for _, e := range sync.Get("presence").Get("events").Array() {
 					t.Logf("lllll %s", e)
@@ -399,7 +405,9 @@ func TestPartialStateJoin(t *testing.T) {
 
 		// Alice should see Derek's to-device message when she syncs.
 		alice.MustSyncUntil(t,
-			client.SyncReq{},
+			client.SyncReq{
+				Filter: buildLazyLoadingSyncFilter(nil),
+			},
 			func(userID string, sync gjson.Result) error {
 				for _, e := range sync.Get("to_device.events").Array() {
 					if e.Get("sender").Str == derekUserId &&

From 5b7f8f495f47e23580416456b8bcd23ca037cdb0 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Fri, 21 Oct 2022 18:40:44 +0100
Subject: [PATCH 14/20] fixup to-device check condition

---
 tests/federation_room_join_partial_state_test.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/federation_room_join_partial_state_test.go b/tests/federation_room_join_partial_state_test.go
index 740c625b..10aa2cf8 100644
--- a/tests/federation_room_join_partial_state_test.go
+++ b/tests/federation_room_join_partial_state_test.go
@@ -411,8 +411,7 @@ func TestPartialStateJoin(t *testing.T) {
 			func(userID string, sync gjson.Result) error {
 				for _, e := range sync.Get("to_device.events").Array() {
 					if e.Get("sender").Str == derekUserId &&
-						e.Get("type").Str == "m.test" &&
-						e.Get("message_id").Str == messageId {
+						e.Get("type").Str == "m.test" {
 						return nil
 					}
 				}

From 73b2398be32a31c1c1881666280652714e0b387a Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Mon, 24 Oct 2022 12:49:32 +0100
Subject: [PATCH 15/20] Use incremental sync in typing notif test

---
 tests/federation_room_join_partial_state_test.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/federation_room_join_partial_state_test.go b/tests/federation_room_join_partial_state_test.go
index 10aa2cf8..59437e8a 100644
--- a/tests/federation_room_join_partial_state_test.go
+++ b/tests/federation_room_join_partial_state_test.go
@@ -269,7 +269,7 @@ func TestPartialStateJoin(t *testing.T) {
 		psjResult.Server.MustSendTransaction(t, deployment, "hs1", []json.RawMessage{}, []gomatrixserverlib.EDU{edu})
 
 		// Alice should be able to see that Derek is typing (even though HS1 is resyncing).
-		alice.MustSyncUntil(t,
+		aliceNextBatch := alice.MustSyncUntil(t,
 			client.SyncReq{
 				Filter: buildLazyLoadingSyncFilter(nil),
 			},
@@ -312,6 +312,7 @@ func TestPartialStateJoin(t *testing.T) {
 		alice.MustSyncUntil(t,
 			client.SyncReq{
 				Filter: buildLazyLoadingSyncFilter(nil),
+				Since: aliceNextBatch,
 			},
 			client.SyncEphemeralHas(serverRoom.RoomID, func(result gjson.Result) bool {
 				return (result.Get("type").Str == "m.typing" &&

From 6d969defa9877315628aa364fcdf409133ced15a Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Mon, 24 Oct 2022 19:35:48 +0100
Subject: [PATCH 16/20] Fix typo

Co-authored-by: Erik Johnston <erik@matrix.org>
---
 tests/federation_room_join_partial_state_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/federation_room_join_partial_state_test.go b/tests/federation_room_join_partial_state_test.go
index 59437e8a..29d6546c 100644
--- a/tests/federation_room_join_partial_state_test.go
+++ b/tests/federation_room_join_partial_state_test.go
@@ -220,7 +220,7 @@ func TestPartialStateJoin(t *testing.T) {
 
 	// we should be able to send events in the room, during the resync
 	t.Run("CanSendEventsDuringPartialStateJoin", func(t *testing.T) {
-		// See https://github.com/matrix-org/synapse/issues/12997")
+		// See https://github.com/matrix-org/synapse/issues/12997
 		t.Skip("Cannot yet send events during resync")
 		alice := deployment.RegisterUser(t, "hs1", "t3alice", "secret", false)
 

From 7570ad3edc986f8a59eb89044ccd3e9f91afbfd1 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Mon, 24 Oct 2022 19:37:12 +0100
Subject: [PATCH 17/20] Extra use of `aliceNextBatch`

---
 tests/federation_room_join_partial_state_test.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/federation_room_join_partial_state_test.go b/tests/federation_room_join_partial_state_test.go
index 29d6546c..578e7e02 100644
--- a/tests/federation_room_join_partial_state_test.go
+++ b/tests/federation_room_join_partial_state_test.go
@@ -291,7 +291,7 @@ func TestPartialStateJoin(t *testing.T) {
 		event := psjResult.CreateMessageEvent(t, "charlie", nil)
 		serverRoom.AddEvent(event)
 		server.MustSendTransaction(t, deployment, "hs1", []json.RawMessage{event.JSON()}, nil)
-		awaitEventViaSync(t, alice, serverRoom.RoomID, event.EventID(), "")
+		aliceNextBatch = awaitEventViaSync(t, alice, serverRoom.RoomID, event.EventID(), aliceNextBatch)
 
 		// The resync completes.
 		psjResult.FinishStateRequest()
@@ -312,7 +312,7 @@ func TestPartialStateJoin(t *testing.T) {
 		alice.MustSyncUntil(t,
 			client.SyncReq{
 				Filter: buildLazyLoadingSyncFilter(nil),
-				Since: aliceNextBatch,
+				Since:  aliceNextBatch,
 			},
 			client.SyncEphemeralHas(serverRoom.RoomID, func(result gjson.Result) bool {
 				return (result.Get("type").Str == "m.typing" &&

From 21646b51e62c25196ed978b6eb76d7f1c6ff95ff Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Mon, 24 Oct 2022 19:41:40 +0100
Subject: [PATCH 18/20] remove debug oops

---
 tests/federation_room_join_partial_state_test.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/federation_room_join_partial_state_test.go b/tests/federation_room_join_partial_state_test.go
index 578e7e02..c686afcd 100644
--- a/tests/federation_room_join_partial_state_test.go
+++ b/tests/federation_room_join_partial_state_test.go
@@ -360,7 +360,6 @@ func TestPartialStateJoin(t *testing.T) {
 			},
 			func(userID string, sync gjson.Result) error {
 				for _, e := range sync.Get("presence").Get("events").Array() {
-					t.Logf("lllll %s", e)
 					if e.Get("sender").Str == derekUserId {
 						return nil
 					}

From 23d422d065ae6c3ddd4c6413c8a4b1199fd62dfe Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 25 Oct 2022 12:50:53 +0100
Subject: [PATCH 19/20] Try to test that Alice sees device list changes

---
 ...federation_room_join_partial_state_test.go | 20 +++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/tests/federation_room_join_partial_state_test.go b/tests/federation_room_join_partial_state_test.go
index c686afcd..446bd8f1 100644
--- a/tests/federation_room_join_partial_state_test.go
+++ b/tests/federation_room_join_partial_state_test.go
@@ -498,10 +498,26 @@ func TestPartialStateJoin(t *testing.T) {
 			Type:    "m.device_list_update",
 			Content: content,
 		}
+		aliceNextBatch := getSyncToken(t, alice)
 		psjResult.Server.MustSendTransaction(t, deployment, "hs1", []json.RawMessage{}, []gomatrixserverlib.EDU{edu})
 
-		// If we want to check the sync we need to have an encrypted room,
-		// for now just check that the fed transaction is accepted.
+		// Check that Alice is told that Derek's devices have changed.
+		aliceNextBatch = alice.MustSyncUntil(
+			t,
+			client.SyncReq{
+				Filter: buildLazyLoadingSyncFilter(nil),
+				Since:  aliceNextBatch,
+			},
+			func(clientUserID string, res gjson.Result) error {
+				matcher := match.JSONCheckOff(
+					"device_lists.changed",
+					[]interface{}{derekUserId},
+					func(r gjson.Result) interface{} { return r },
+					nil,
+				)
+				return matcher([]byte(res.Raw))
+			},
+		)
 	})
 
 	// we should be able to receive signing key update EDU over federation during the resync

From d2c2bc588135b6dbced4740dca31f041ba5f1d13 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 25 Oct 2022 13:53:11 +0100
Subject: [PATCH 20/20] Only get device lists after resync

---
 tests/federation_room_join_partial_state_test.go | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tests/federation_room_join_partial_state_test.go b/tests/federation_room_join_partial_state_test.go
index 446bd8f1..a08830f0 100644
--- a/tests/federation_room_join_partial_state_test.go
+++ b/tests/federation_room_join_partial_state_test.go
@@ -501,7 +501,12 @@ func TestPartialStateJoin(t *testing.T) {
 		aliceNextBatch := getSyncToken(t, alice)
 		psjResult.Server.MustSendTransaction(t, deployment, "hs1", []json.RawMessage{}, []gomatrixserverlib.EDU{edu})
 
+		// The resync completes.
+		psjResult.FinishStateRequest()
+
 		// Check that Alice is told that Derek's devices have changed.
+		// (Alice does not get told this during the resync, since we can't know
+		// for certain who is in that room until the resync completes.)
 		aliceNextBatch = alice.MustSyncUntil(
 			t,
 			client.SyncReq{
@@ -512,7 +517,7 @@ func TestPartialStateJoin(t *testing.T) {
 				matcher := match.JSONCheckOff(
 					"device_lists.changed",
 					[]interface{}{derekUserId},
-					func(r gjson.Result) interface{} { return r },
+					func(r gjson.Result) interface{} { return r.Str },
 					nil,
 				)
 				return matcher([]byte(res.Raw))