Skip to content
This repository has been archived by the owner on Apr 26, 2024. It is now read-only.

Clean up tests.test_visibility to remove legacy code. #11495

Merged
merged 10 commits into from
Dec 2, 2021
Merged

Clean up tests.test_visibility to remove legacy code. #11495

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
ed19520
Remove use of setup_test_homeserver and use HomeserverTestCase instead
reivilibre Dec 2, 2021
c713e16
Remove use of inlineCallbacks
reivilibre Dec 2, 2021
20b1ce9
Rename test helpers
reivilibre Dec 2, 2021
60b1515
Annotate fields
reivilibre Dec 2, 2021
83d0ec4
Annotate some methods
reivilibre Dec 2, 2021
e213bdc
Fix the typing but not the functionality of test_large_room
reivilibre Dec 2, 2021
5ce6258
Take test_visibility off the mypy list
reivilibre Dec 2, 2021
8c42109
Newsfile
reivilibre Dec 2, 2021
5c57265
Remove unloved test_large_room test, which is broken and seems to hav…
reivilibre Dec 2, 2021
4469113
Gah, isort!
reivilibre Dec 2, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/11495.misc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Clean up `tests.test_visibility` to remove legacy code.
1 change: 0 additions & 1 deletion mypy.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,6 @@ exclude = (?x)
|tests/test_server.py
|tests/test_state.py
|tests/test_terms_auth.py
|tests/test_visibility.py
|tests/unittest.py
|tests/util/caches/test_cached_call.py
|tests/util/caches/test_deferred_cache.py
Expand Down
241 changes: 39 additions & 202 deletions tests/test_visibility.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,35 +13,30 @@
# limitations under the License.
import logging
from typing import Optional
from unittest.mock import Mock

from twisted.internet import defer
from twisted.internet.defer import succeed

from synapse.api.room_versions import RoomVersions
from synapse.events import FrozenEvent
from synapse.events import EventBase
from synapse.types import JsonDict
from synapse.visibility import filter_events_for_server

import tests.unittest
from tests.utils import create_room, setup_test_homeserver
from tests import unittest
from tests.utils import create_room

logger = logging.getLogger(__name__)

TEST_ROOM_ID = "!TEST:ROOM"


class FilterEventsForServerTestCase(tests.unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
self.hs = yield setup_test_homeserver(self.addCleanup)
class FilterEventsForServerTestCase(unittest.HomeserverTestCase):
def setUp(self) -> None:
super(FilterEventsForServerTestCase, self).setUp()
self.event_creation_handler = self.hs.get_event_creation_handler()
self.event_builder_factory = self.hs.get_event_builder_factory()
self.storage = self.hs.get_storage()

yield defer.ensureDeferred(create_room(self.hs, TEST_ROOM_ID, "@someone:ROOM"))
self.get_success(create_room(self.hs, TEST_ROOM_ID, "@someone:ROOM"))

@defer.inlineCallbacks
def test_filtering(self):
def test_filtering(self) -> None:
#
# The events to be filtered consist of 10 membership events (it doesn't
# really matter if they are joins or leaves, so let's make them joins).
Expand All @@ -51,18 +46,20 @@ def test_filtering(self):
#

# before we do that, we persist some other events to act as state.
yield self.inject_visibility("@admin:hs", "joined")
self.get_success(self._inject_visibility("@admin:hs", "joined"))
for i in range(0, 10):
yield self.inject_room_member("@resident%i:hs" % i)
self.get_success(self._inject_room_member("@resident%i:hs" % i))

events_to_filter = []

for i in range(0, 10):
user = "@user%i:%s" % (i, "test_server" if i == 5 else "other_server")
evt = yield self.inject_room_member(user, extra_content={"a": "b"})
evt = self.get_success(
self._inject_room_member(user, extra_content={"a": "b"})
)
events_to_filter.append(evt)

filtered = yield defer.ensureDeferred(
filtered = self.get_success(
filter_events_for_server(self.storage, "test_server", events_to_filter)
)

Expand All @@ -75,34 +72,31 @@ def test_filtering(self):
self.assertEqual(events_to_filter[i].event_id, filtered[i].event_id)
self.assertEqual(filtered[i].content["a"], "b")

@defer.inlineCallbacks
def test_erased_user(self):
def test_erased_user(self) -> None:
# 4 message events, from erased and unerased users, with a membership
# change in the middle of them.
events_to_filter = []

evt = yield self.inject_message("@unerased:local_hs")
evt = self.get_success(self._inject_message("@unerased:local_hs"))
events_to_filter.append(evt)

evt = yield self.inject_message("@erased:local_hs")
evt = self.get_success(self._inject_message("@erased:local_hs"))
events_to_filter.append(evt)

evt = yield self.inject_room_member("@joiner:remote_hs")
evt = self.get_success(self._inject_room_member("@joiner:remote_hs"))
events_to_filter.append(evt)

evt = yield self.inject_message("@unerased:local_hs")
evt = self.get_success(self._inject_message("@unerased:local_hs"))
events_to_filter.append(evt)

evt = yield self.inject_message("@erased:local_hs")
evt = self.get_success(self._inject_message("@erased:local_hs"))
events_to_filter.append(evt)

# the erasey user gets erased
yield defer.ensureDeferred(
self.hs.get_datastore().mark_user_erased("@erased:local_hs")
)
self.get_success(self.hs.get_datastore().mark_user_erased("@erased:local_hs"))

# ... and the filtering happens.
filtered = yield defer.ensureDeferred(
filtered = self.get_success(
filter_events_for_server(self.storage, "test_server", events_to_filter)
)

Expand All @@ -123,8 +117,7 @@ def test_erased_user(self):
for i in (1, 4):
self.assertNotIn("body", filtered[i].content)

@defer.inlineCallbacks
def inject_visibility(self, user_id, visibility):
def _inject_visibility(self, user_id: str, visibility: str) -> EventBase:
content = {"history_visibility": visibility}
builder = self.event_builder_factory.for_room_version(
RoomVersions.V1,
Expand All @@ -137,18 +130,18 @@ def inject_visibility(self, user_id, visibility):
},
)

event, context = yield defer.ensureDeferred(
event, context = self.get_success(
self.event_creation_handler.create_new_client_event(builder)
)
yield defer.ensureDeferred(
self.storage.persistence.persist_event(event, context)
)
self.get_success(self.storage.persistence.persist_event(event, context))
return event

@defer.inlineCallbacks
def inject_room_member(
self, user_id, membership="join", extra_content: Optional[dict] = None
):
def _inject_room_member(
self,
user_id: str,
membership: str = "join",
extra_content: Optional[JsonDict] = None,
) -> EventBase:
content = {"membership": membership}
content.update(extra_content or {})
builder = self.event_builder_factory.for_room_version(
Expand All @@ -162,17 +155,16 @@ def inject_room_member(
},
)

event, context = yield defer.ensureDeferred(
event, context = self.get_success(
self.event_creation_handler.create_new_client_event(builder)
)

yield defer.ensureDeferred(
self.storage.persistence.persist_event(event, context)
)
self.get_success(self.storage.persistence.persist_event(event, context))
return event

@defer.inlineCallbacks
def inject_message(self, user_id, content=None):
def _inject_message(
self, user_id: str, content: Optional[JsonDict] = None
) -> EventBase:
if content is None:
content = {"body": "testytest", "msgtype": "m.text"}
builder = self.event_builder_factory.for_room_version(
Expand All @@ -185,164 +177,9 @@ def inject_message(self, user_id, content=None):
},
)

event, context = yield defer.ensureDeferred(
event, context = self.get_success(
self.event_creation_handler.create_new_client_event(builder)
)

yield defer.ensureDeferred(
self.storage.persistence.persist_event(event, context)
)
self.get_success(self.storage.persistence.persist_event(event, context))
return event

@defer.inlineCallbacks
def test_large_room(self):
# see what happens when we have a large room with hundreds of thousands
# of membership events

# As above, the events to be filtered consist of 10 membership events,
# where one of them is for a user on the server we are filtering for.

import cProfile
import pstats
import time

# we stub out the store, because building up all that state the normal
# way is very slow.
test_store = _TestStore()

# our initial state is 100000 membership events and one
# history_visibility event.
room_state = []

history_visibility_evt = FrozenEvent(
{
"event_id": "$history_vis",
"type": "m.room.history_visibility",
"sender": "@resident_user_0:test.com",
"state_key": "",
"room_id": TEST_ROOM_ID,
"content": {"history_visibility": "joined"},
}
)
room_state.append(history_visibility_evt)
test_store.add_event(history_visibility_evt)

for i in range(0, 100000):
user = "@resident_user_%i:test.com" % (i,)
evt = FrozenEvent(
{
"event_id": "$res_event_%i" % (i,),
"type": "m.room.member",
"state_key": user,
"sender": user,
"room_id": TEST_ROOM_ID,
"content": {"membership": "join", "extra": "zzz,"},
}
)
room_state.append(evt)
test_store.add_event(evt)

events_to_filter = []
for i in range(0, 10):
user = "@user%i:%s" % (i, "test_server" if i == 5 else "other_server")
evt = FrozenEvent(
{
"event_id": "$evt%i" % (i,),
"type": "m.room.member",
"state_key": user,
"sender": user,
"room_id": TEST_ROOM_ID,
"content": {"membership": "join", "extra": "zzz"},
}
)
events_to_filter.append(evt)
room_state.append(evt)

test_store.add_event(evt)
test_store.set_state_ids_for_event(
evt, {(e.type, e.state_key): e.event_id for e in room_state}
)

pr = cProfile.Profile()
pr.enable()

logger.info("Starting filtering")
start = time.time()

storage = Mock()
storage.main = test_store
storage.state = test_store

filtered = yield defer.ensureDeferred(
filter_events_for_server(test_store, "test_server", events_to_filter)
)
logger.info("Filtering took %f seconds", time.time() - start)

pr.disable()
with open("filter_events_for_server.profile", "w+") as f:
ps = pstats.Stats(pr, stream=f).sort_stats("cumulative")
ps.print_stats()

# the result should be 5 redacted events, and 5 unredacted events.
for i in range(0, 5):
self.assertEqual(events_to_filter[i].event_id, filtered[i].event_id)
self.assertNotIn("extra", filtered[i].content)

for i in range(5, 10):
self.assertEqual(events_to_filter[i].event_id, filtered[i].event_id)
self.assertEqual(filtered[i].content["extra"], "zzz")

test_large_room.skip = "Disabled by default because it's slow"


class _TestStore:
"""Implements a few methods of the DataStore, so that we can test
filter_events_for_server

"""

def __init__(self):
# data for get_events: a map from event_id to event
self.events = {}

# data for get_state_ids_for_events mock: a map from event_id to
# a map from (type_state_key) -> event_id for the state at that
# event
self.state_ids_for_events = {}

def add_event(self, event):
self.events[event.event_id] = event

def set_state_ids_for_event(self, event, state):
self.state_ids_for_events[event.event_id] = state

def get_state_ids_for_events(self, events, types):
res = {}
include_memberships = False
for (type, state_key) in types:
if type == "m.room.history_visibility":
continue
if type != "m.room.member" or state_key is not None:
raise RuntimeError(
"Unimplemented: get_state_ids with type (%s, %s)"
% (type, state_key)
)
include_memberships = True

if include_memberships:
for event_id in events:
res[event_id] = self.state_ids_for_events[event_id]

else:
k = ("m.room.history_visibility", "")
for event_id in events:
hve = self.state_ids_for_events[event_id][k]
res[event_id] = {k: hve}

return succeed(res)

def get_events(self, events):
return succeed({event_id: self.events[event_id] for event_id in events})

def are_users_erased(self, users):
return succeed({u: False for u in users})