Skip to content
This repository was archived by the owner on Apr 26, 2024. It is now read-only.

Commit 888a29f

Browse files
erikjohnstonerikjohnston
authored
Wait for lazy join to complete when getting current state (#12872)
1 parent 782cb74 commit 888a29f

33 files changed

+361
-82
lines changed

‎changelog.d/12872.misc

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Faster room joins: when querying the current state of the room, wait for state to be populated.

‎synapse/events/third_party_rules.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,7 @@ def __init__(self, hs: "HomeServer"):
152152
self.third_party_rules = None
153153

154154
self.store = hs.get_datastores().main
155+
self._storage_controllers = hs.get_storage_controllers()
155156

156157
self._check_event_allowed_callbacks: List[CHECK_EVENT_ALLOWED_CALLBACK] = []
157158
self._on_create_room_callbacks: List[ON_CREATE_ROOM_CALLBACK] = []
@@ -463,7 +464,7 @@ async def _get_state_map_for_room(self, room_id: str) -> StateMap[EventBase]:
463464
Returns:
464465
A dict mapping (event type, state key) to state event.
465466
"""
466-
state_ids = await self.store.get_filtered_current_state_ids(room_id)
467+
state_ids = await self._storage_controllers.state.get_current_state_ids(room_id)
467468
room_state_events = await self.store.get_events(state_ids.values())
468469

469470
state_events = {}

‎synapse/federation/federation_server.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,8 @@ def __init__(self, hs: "HomeServer"):
118118
self.state = hs.get_state_handler()
119119
self._event_auth_handler = hs.get_event_auth_handler()
120120

121+
self._state_storage_controller = hs.get_storage_controllers().state
122+
121123
self.device_handler = hs.get_device_handler()
122124

123125
# Ensure the following handlers are loaded since they register callbacks
@@ -1221,7 +1223,7 @@ async def check_server_matches_acl(self, server_name: str, room_id: str) -> None
12211223
Raises:
12221224
AuthError if the server does not match the ACL
12231225
"""
1224-
state_ids = await self.store.get_current_state_ids(room_id)
1226+
state_ids = await self._state_storage_controller.get_current_state_ids(room_id)
12251227
acl_event_id = state_ids.get((EventTypes.ServerACL, ""))
12261228

12271229
if not acl_event_id:

‎synapse/handlers/device.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,7 @@ async def get_user_ids_changed(
166166
possibly_changed = set(changed)
167167
possibly_left = set()
168168
for room_id in rooms_changed:
169-
current_state_ids = await self.store.get_current_state_ids(room_id)
169+
current_state_ids = await self._state_storage.get_current_state_ids(room_id)
170170

171171
# The user may have left the room
172172
# TODO: Check if they actually did or if we were just invited.

‎synapse/handlers/directory.py

+6-1
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ def __init__(self, hs: "HomeServer"):
4545
self.appservice_handler = hs.get_application_service_handler()
4646
self.event_creation_handler = hs.get_event_creation_handler()
4747
self.store = hs.get_datastores().main
48+
self._storage_controllers = hs.get_storage_controllers()
4849
self.config = hs.config
4950
self.enable_room_list_search = hs.config.roomdirectory.enable_room_list_search
5051
self.require_membership = hs.config.server.require_membership_for_aliases
@@ -463,7 +464,11 @@ async def edit_published_room_list(
463464
making_public = visibility == "public"
464465
if making_public:
465466
room_aliases = await self.store.get_aliases_for_room(room_id)
466-
canonical_alias = await self.store.get_canonical_alias_for_room(room_id)
467+
canonical_alias = (
468+
await self._storage_controllers.state.get_canonical_alias_for_room(
469+
room_id
470+
)
471+
)
467472
if canonical_alias:
468473
room_aliases.append(canonical_alias)
469474

‎synapse/handlers/federation.py

+6-1
Original file line numberDiff line numberDiff line change
@@ -750,7 +750,9 @@ async def on_make_join_request(
750750
# Note that this requires the /send_join request to come back to the
751751
# same server.
752752
if room_version.msc3083_join_rules:
753-
state_ids = await self.store.get_current_state_ids(room_id)
753+
state_ids = await self._state_storage_controller.get_current_state_ids(
754+
room_id
755+
)
754756
if await self._event_auth_handler.has_restricted_join_rules(
755757
state_ids, room_version
756758
):
@@ -1552,6 +1554,9 @@ async def _sync_partial_state_room(
15521554
success = await self.store.clear_partial_state_room(room_id)
15531555
if success:
15541556
logger.info("State resync complete for %s", room_id)
1557+
self._storage_controllers.state.notify_room_un_partial_stated(
1558+
room_id
1559+
)
15551560

15561561
# TODO(faster_joins) update room stats and user directory?
15571562
return

‎synapse/handlers/message.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,7 @@ async def get_state_events(
217217
)
218218

219219
if membership == Membership.JOIN:
220-
state_ids = await self.store.get_filtered_current_state_ids(
220+
state_ids = await self._state_storage_controller.get_current_state_ids(
221221
room_id, state_filter=state_filter
222222
)
223223
room_state = await self.store.get_events(state_ids.values())

‎synapse/handlers/presence.py

+5-1
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,7 @@ class BasePresenceHandler(abc.ABC):
134134
def __init__(self, hs: "HomeServer"):
135135
self.clock = hs.get_clock()
136136
self.store = hs.get_datastores().main
137+
self._storage_controllers = hs.get_storage_controllers()
137138
self.presence_router = hs.get_presence_router()
138139
self.state = hs.get_state_handler()
139140
self.is_mine_id = hs.is_mine_id
@@ -1348,7 +1349,10 @@ async def _unsafe_process(self) -> None:
13481349
self._event_pos,
13491350
room_max_stream_ordering,
13501351
)
1351-
max_pos, deltas = await self.store.get_current_state_deltas(
1352+
(
1353+
max_pos,
1354+
deltas,
1355+
) = await self._storage_controllers.state.get_current_state_deltas(
13521356
self._event_pos, room_max_stream_ordering
13531357
)
13541358

‎synapse/handlers/register.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ class LoginDict(TypedDict):
8787
class RegistrationHandler:
8888
def __init__(self, hs: "HomeServer"):
8989
self.store = hs.get_datastores().main
90+
self._storage_controllers = hs.get_storage_controllers()
9091
self.clock = hs.get_clock()
9192
self.hs = hs
9293
self.auth = hs.get_auth()
@@ -528,7 +529,7 @@ async def _join_rooms(self, user_id: str) -> None:
528529

529530
if requires_invite:
530531
# If the server is in the room, check if the room is public.
531-
state = await self.store.get_filtered_current_state_ids(
532+
state = await self._storage_controllers.state.get_current_state_ids(
532533
room_id, StateFilter.from_types([(EventTypes.JoinRules, "")])
533534
)
534535

‎synapse/handlers/room.py

+9-4
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ class EventContext:
107107
class RoomCreationHandler:
108108
def __init__(self, hs: "HomeServer"):
109109
self.store = hs.get_datastores().main
110+
self._storage_controllers = hs.get_storage_controllers()
110111
self.auth = hs.get_auth()
111112
self.clock = hs.get_clock()
112113
self.hs = hs
@@ -480,8 +481,10 @@ async def clone_existing_room(
480481
if room_type == RoomTypes.SPACE:
481482
types_to_copy.append((EventTypes.SpaceChild, None))
482483

483-
old_room_state_ids = await self.store.get_filtered_current_state_ids(
484-
old_room_id, StateFilter.from_types(types_to_copy)
484+
old_room_state_ids = (
485+
await self._storage_controllers.state.get_current_state_ids(
486+
old_room_id, StateFilter.from_types(types_to_copy)
487+
)
485488
)
486489
# map from event_id to BaseEvent
487490
old_room_state_events = await self.store.get_events(old_room_state_ids.values())
@@ -558,8 +561,10 @@ async def clone_existing_room(
558561
)
559562

560563
# Transfer membership events
561-
old_room_member_state_ids = await self.store.get_filtered_current_state_ids(
562-
old_room_id, StateFilter.from_types([(EventTypes.Member, None)])
564+
old_room_member_state_ids = (
565+
await self._storage_controllers.state.get_current_state_ids(
566+
old_room_id, StateFilter.from_types([(EventTypes.Member, None)])
567+
)
563568
)
564569

565570
# map from event_id to BaseEvent

‎synapse/handlers/room_list.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
class RoomListHandler:
5151
def __init__(self, hs: "HomeServer"):
5252
self.store = hs.get_datastores().main
53+
self._storage_controllers = hs.get_storage_controllers()
5354
self.hs = hs
5455
self.enable_room_list_search = hs.config.roomdirectory.enable_room_list_search
5556
self.response_cache: ResponseCache[
@@ -274,7 +275,7 @@ async def generate_room_entry(
274275
if aliases:
275276
result["aliases"] = aliases
276277

277-
current_state_ids = await self.store.get_current_state_ids(
278+
current_state_ids = await self._storage_controllers.state.get_current_state_ids(
278279
room_id, on_invalidate=cache_context.invalidate
279280
)
280281

‎synapse/handlers/room_member.py

+4-1
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
6868
def __init__(self, hs: "HomeServer"):
6969
self.hs = hs
7070
self.store = hs.get_datastores().main
71+
self._storage_controllers = hs.get_storage_controllers()
7172
self.auth = hs.get_auth()
7273
self.state_handler = hs.get_state_handler()
7374
self.config = hs.config
@@ -994,7 +995,9 @@ async def _should_perform_remote_join(
994995
# If the host is in the room, but not one of the authorised hosts
995996
# for restricted join rules, a remote join must be used.
996997
room_version = await self.store.get_room_version(room_id)
997-
current_state_ids = await self.store.get_current_state_ids(room_id)
998+
current_state_ids = await self._storage_controllers.state.get_current_state_ids(
999+
room_id
1000+
)
9981001

9991002
# If restricted join rules are not being used, a local join can always
10001003
# be used.

‎synapse/handlers/room_summary.py

+8-3
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ class RoomSummaryHandler:
9090
def __init__(self, hs: "HomeServer"):
9191
self._event_auth_handler = hs.get_event_auth_handler()
9292
self._store = hs.get_datastores().main
93+
self._storage_controllers = hs.get_storage_controllers()
9394
self._event_serializer = hs.get_event_client_serializer()
9495
self._server_name = hs.hostname
9596
self._federation_client = hs.get_federation_client()
@@ -537,7 +538,7 @@ async def _is_local_room_accessible(
537538
Returns:
538539
True if the room is accessible to the requesting user or server.
539540
"""
540-
state_ids = await self._store.get_current_state_ids(room_id)
541+
state_ids = await self._storage_controllers.state.get_current_state_ids(room_id)
541542

542543
# If there's no state for the room, it isn't known.
543544
if not state_ids:
@@ -702,7 +703,9 @@ async def _build_room_entry(self, room_id: str, for_federation: bool) -> JsonDic
702703
# there should always be an entry
703704
assert stats is not None, "unable to retrieve stats for %s" % (room_id,)
704705

705-
current_state_ids = await self._store.get_current_state_ids(room_id)
706+
current_state_ids = await self._storage_controllers.state.get_current_state_ids(
707+
room_id
708+
)
706709
create_event = await self._store.get_event(
707710
current_state_ids[(EventTypes.Create, "")]
708711
)
@@ -760,7 +763,9 @@ async def _get_child_events(self, room_id: str) -> Iterable[EventBase]:
760763
"""
761764

762765
# look for child rooms/spaces.
763-
current_state_ids = await self._store.get_current_state_ids(room_id)
766+
current_state_ids = await self._storage_controllers.state.get_current_state_ids(
767+
room_id
768+
)
764769

765770
events = await self._store.get_events_as_list(
766771
[

‎synapse/handlers/stats.py

+5-1
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ class StatsHandler:
4040
def __init__(self, hs: "HomeServer"):
4141
self.hs = hs
4242
self.store = hs.get_datastores().main
43+
self._storage_controllers = hs.get_storage_controllers()
4344
self.state = hs.get_state_handler()
4445
self.server_name = hs.hostname
4546
self.clock = hs.get_clock()
@@ -105,7 +106,10 @@ async def _unsafe_process(self) -> None:
105106
logger.debug(
106107
"Processing room stats %s->%s", self.pos, room_max_stream_ordering
107108
)
108-
max_pos, deltas = await self.store.get_current_state_deltas(
109+
(
110+
max_pos,
111+
deltas,
112+
) = await self._storage_controllers.state.get_current_state_deltas(
109113
self.pos, room_max_stream_ordering
110114
)
111115

‎synapse/handlers/sync.py

+9-4
Original file line numberDiff line numberDiff line change
@@ -506,8 +506,10 @@ async def _load_filtered_recents(
506506
# ensure that we always include current state in the timeline
507507
current_state_ids: FrozenSet[str] = frozenset()
508508
if any(e.is_state() for e in recents):
509-
current_state_ids_map = await self.store.get_current_state_ids(
510-
room_id
509+
current_state_ids_map = (
510+
await self._state_storage_controller.get_current_state_ids(
511+
room_id
512+
)
511513
)
512514
current_state_ids = frozenset(current_state_ids_map.values())
513515

@@ -574,8 +576,11 @@ async def _load_filtered_recents(
574576
# ensure that we always include current state in the timeline
575577
current_state_ids = frozenset()
576578
if any(e.is_state() for e in loaded_recents):
577-
current_state_ids_map = await self.store.get_current_state_ids(
578-
room_id
579+
# FIXME(faster_joins): We use the partial state here as
580+
# we don't want to block `/sync` on finishing a lazy join.
581+
# Is this the correct way of doing it?
582+
current_state_ids_map = (
583+
await self.store.get_partial_current_state_ids(room_id)
579584
)
580585
current_state_ids = frozenset(current_state_ids_map.values())
581586

‎synapse/handlers/user_directory.py

+5-1
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ def __init__(self, hs: "HomeServer"):
5656
super().__init__(hs)
5757

5858
self.store = hs.get_datastores().main
59+
self._storage_controllers = hs.get_storage_controllers()
5960
self.server_name = hs.hostname
6061
self.clock = hs.get_clock()
6162
self.notifier = hs.get_notifier()
@@ -174,7 +175,10 @@ async def _unsafe_process(self) -> None:
174175
logger.debug(
175176
"Processing user stats %s->%s", self.pos, room_max_stream_ordering
176177
)
177-
max_pos, deltas = await self.store.get_current_state_deltas(
178+
(
179+
max_pos,
180+
deltas,
181+
) = await self._storage_controllers.state.get_current_state_deltas(
178182
self.pos, room_max_stream_ordering
179183
)
180184

‎synapse/module_api/__init__.py

+8-11
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,7 @@ def __init__(self, hs: "HomeServer", auth_handler: AuthHandler) -> None:
194194
self._store: Union[
195195
DataStore, "GenericWorkerSlavedStore"
196196
] = hs.get_datastores().main
197+
self._storage_controllers = hs.get_storage_controllers()
197198
self._auth = hs.get_auth()
198199
self._auth_handler = auth_handler
199200
self._server_name = hs.hostname
@@ -911,7 +912,7 @@ def get_state_events_in_room(
911912
The filtered state events in the room.
912913
"""
913914
state_ids = yield defer.ensureDeferred(
914-
self._store.get_filtered_current_state_ids(
915+
self._storage_controllers.state.get_current_state_ids(
915916
room_id=room_id, state_filter=StateFilter.from_types(types)
916917
)
917918
)
@@ -1289,20 +1290,16 @@ async def get_room_state(
12891290
# regardless of their state key
12901291
]
12911292
"""
1293+
state_filter = None
12921294
if event_filter:
12931295
# If a filter was provided, turn it into a StateFilter and retrieve a filtered
12941296
# view of the state.
12951297
state_filter = StateFilter.from_types(event_filter)
1296-
state_ids = await self._store.get_filtered_current_state_ids(
1297-
room_id,
1298-
state_filter,
1299-
)
1300-
else:
1301-
# If no filter was provided, get the whole state. We could also reuse the call
1302-
# to get_filtered_current_state_ids above, with `state_filter = StateFilter.all()`,
1303-
# but get_filtered_current_state_ids isn't cached and `get_current_state_ids`
1304-
# is, so using the latter when we can is better for perf.
1305-
state_ids = await self._store.get_current_state_ids(room_id)
1298+
1299+
state_ids = await self._storage_controllers.state.get_current_state_ids(
1300+
room_id,
1301+
state_filter,
1302+
)
13061303

13071304
state_events = await self._store.get_events(state_ids.values())
13081305

‎synapse/push/mailer.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -255,7 +255,9 @@ async def send_notification_mail(
255255
user_display_name = user_id
256256

257257
async def _fetch_room_state(room_id: str) -> None:
258-
room_state = await self.store.get_current_state_ids(room_id)
258+
room_state = await self._state_storage_controller.get_current_state_ids(
259+
room_id
260+
)
259261
state_by_room[room_id] = room_state
260262

261263
# Run at most 3 of these at once: sync does 10 at a time but email

‎synapse/rest/admin/rooms.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -418,6 +418,7 @@ class RoomStateRestServlet(RestServlet):
418418
def __init__(self, hs: "HomeServer"):
419419
self.auth = hs.get_auth()
420420
self.store = hs.get_datastores().main
421+
self._storage_controllers = hs.get_storage_controllers()
421422
self.clock = hs.get_clock()
422423
self._event_serializer = hs.get_event_client_serializer()
423424

@@ -430,7 +431,7 @@ async def on_GET(
430431
if not ret:
431432
raise NotFoundError("Room not found")
432433

433-
event_ids = await self.store.get_current_state_ids(room_id)
434+
event_ids = await self._storage_controllers.state.get_current_state_ids(room_id)
434435
events = await self.store.get_events(event_ids.values())
435436
now = self.clock.time_msec()
436437
room_state = self._event_serializer.serialize_events(events.values(), now)

‎synapse/storage/_base.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ def _invalidate_state_caches(
7777

7878
# Purge other caches based on room state.
7979
self._attempt_to_invalidate_cache("get_room_summary", (room_id,))
80-
self._attempt_to_invalidate_cache("get_current_state_ids", (room_id,))
80+
self._attempt_to_invalidate_cache("get_partial_current_state_ids", (room_id,))
8181

8282
def _attempt_to_invalidate_cache(
8383
self, cache_name: str, key: Optional[Collection[Any]]

0 commit comments

Comments
 (0)