From 133995ca5d11c6ac48c48398196b2f792f975f45 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Fri, 1 Nov 2024 12:07:59 -0400 Subject: [PATCH 1/3] MatrixRTCSession: handle rate limit errors --- src/matrixrtc/MatrixRTCSession.ts | 51 ++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/src/matrixrtc/MatrixRTCSession.ts b/src/matrixrtc/MatrixRTCSession.ts index 52258ea53a2..4551956bea5 100644 --- a/src/matrixrtc/MatrixRTCSession.ts +++ b/src/matrixrtc/MatrixRTCSession.ts @@ -34,7 +34,7 @@ import { randomString, secureRandomBase64Url } from "../randomstring.ts"; import { EncryptionKeysEventContent } from "./types.ts"; import { decodeBase64, encodeUnpaddedBase64 } from "../base64.ts"; import { KnownMembership } from "../@types/membership.ts"; -import { MatrixError, safeGetRetryAfterMs } from "../http-api/errors.ts"; +import { HTTPError, MatrixError, safeGetRetryAfterMs } from "../http-api/errors.ts"; import { MatrixEvent } from "../models/event.ts"; import { isLivekitFocusActive } from "./LivekitFocus.ts"; import { ExperimentalGroupCallRoomMemberState } from "../webrtc/groupCall.ts"; @@ -1031,7 +1031,7 @@ export class MatrixRTCSession extends TypedEventEmitter => { try { // TODO: If delayed event times out, re-join! - const res = await this.client._unstable_sendDelayedStateEvent( + const res = await resendIfRateLimited(() => this.client._unstable_sendDelayedStateEvent( this.room.roomId, { delay: 8000, @@ -1039,10 +1039,9 @@ export class MatrixRTCSession extends TypedEventEmitter this.client._unstable_updateDelayedEvent( + knownDisconnectDelayId, UpdateDelayedEventAction.Restart, - ); + )); } catch (e) { - // TODO: Make embedded client include errcode, and retry only if not M_NOT_FOUND (or rate-limited) logger.warn("Failed to update delayed disconnection event, prepare it again:", e); this.disconnectDelayId = undefined; await prepareDelayedDisconnection(); @@ -1076,13 +1075,13 @@ export class MatrixRTCSession extends TypedEventEmitter this.client._unstable_updateDelayedEvent( + knownDisconnectDelayId, UpdateDelayedEventAction.Send, - ); + )); sentDelayedDisconnect = true; } catch (e) { - // TODO: Retry if rate-limited logger.error("Failed to send our delayed disconnection event:", e); } this.disconnectDelayId = undefined; @@ -1111,10 +1110,10 @@ export class MatrixRTCSession extends TypedEventEmitter => { try { - await this.client._unstable_updateDelayedEvent(this.disconnectDelayId!, UpdateDelayedEventAction.Restart); + const knownDisconnectDelayId = this.disconnectDelayId!; + await resendIfRateLimited(() => this.client._unstable_updateDelayedEvent(knownDisconnectDelayId, UpdateDelayedEventAction.Restart)); this.scheduleDelayDisconnection(); } catch (e) { - // TODO: Retry if rate-limited logger.error("Failed to delay our disconnection event:", e); } }; @@ -1162,3 +1161,27 @@ export class MatrixRTCSession extends TypedEventEmitter(func: () => Promise, numRetriesAllowed: number = 1): Promise { + while (true) { + try { + return await func(); + } catch (e) { + if (numRetriesAllowed > 0 && e instanceof HTTPError && e.isRateLimitError()) { + numRetriesAllowed--; + let resendDelay: number; + const defaultMs = 5000; + try { + resendDelay = e.getRetryAfterMs() ?? defaultMs; + logger.info(`Rate limited by server, retrying in ${resendDelay}ms`); + } catch (e) { + logger.warn(`Error while retrieving a rate-limit retry delay, retrying after default delay of ${defaultMs}`, e); + resendDelay = defaultMs; + } + await sleep(resendDelay); + } else { + throw e; + } + } + }; +} From 473c3cce6f944505fd6488f542577abb31e467a7 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Fri, 1 Nov 2024 12:11:09 -0400 Subject: [PATCH 2/3] Lint --- src/matrixrtc/MatrixRTCSession.ts | 52 +++++++++++++++++++------------ 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/src/matrixrtc/MatrixRTCSession.ts b/src/matrixrtc/MatrixRTCSession.ts index 4551956bea5..992e50695c0 100644 --- a/src/matrixrtc/MatrixRTCSession.ts +++ b/src/matrixrtc/MatrixRTCSession.ts @@ -1031,15 +1031,17 @@ export class MatrixRTCSession extends TypedEventEmitter => { try { // TODO: If delayed event times out, re-join! - const res = await resendIfRateLimited(() => this.client._unstable_sendDelayedStateEvent( - this.room.roomId, - { - delay: 8000, - }, - EventType.GroupCallMemberPrefix, - {}, // leave event - stateKey, - )); + const res = await resendIfRateLimited(() => + this.client._unstable_sendDelayedStateEvent( + this.room.roomId, + { + delay: 8000, + }, + EventType.GroupCallMemberPrefix, + {}, // leave event + stateKey, + ), + ); this.disconnectDelayId = res.delay_id; } catch (e) { logger.error("Failed to prepare delayed disconnection event:", e); @@ -1058,10 +1060,12 @@ export class MatrixRTCSession extends TypedEventEmitter this.client._unstable_updateDelayedEvent( - knownDisconnectDelayId, - UpdateDelayedEventAction.Restart, - )); + await resendIfRateLimited(() => + this.client._unstable_updateDelayedEvent( + knownDisconnectDelayId, + UpdateDelayedEventAction.Restart, + ), + ); } catch (e) { logger.warn("Failed to update delayed disconnection event, prepare it again:", e); this.disconnectDelayId = undefined; @@ -1076,10 +1080,12 @@ export class MatrixRTCSession extends TypedEventEmitter this.client._unstable_updateDelayedEvent( - knownDisconnectDelayId, - UpdateDelayedEventAction.Send, - )); + await resendIfRateLimited(() => + this.client._unstable_updateDelayedEvent( + knownDisconnectDelayId, + UpdateDelayedEventAction.Send, + ), + ); sentDelayedDisconnect = true; } catch (e) { logger.error("Failed to send our delayed disconnection event:", e); @@ -1111,7 +1117,9 @@ export class MatrixRTCSession extends TypedEventEmitter => { try { const knownDisconnectDelayId = this.disconnectDelayId!; - await resendIfRateLimited(() => this.client._unstable_updateDelayedEvent(knownDisconnectDelayId, UpdateDelayedEventAction.Restart)); + await resendIfRateLimited(() => + this.client._unstable_updateDelayedEvent(knownDisconnectDelayId, UpdateDelayedEventAction.Restart), + ); this.scheduleDelayDisconnection(); } catch (e) { logger.error("Failed to delay our disconnection event:", e); @@ -1163,6 +1171,7 @@ export class MatrixRTCSession extends TypedEventEmitter(func: () => Promise, numRetriesAllowed: number = 1): Promise { + // eslint-disable-next-line no-constant-condition while (true) { try { return await func(); @@ -1175,7 +1184,10 @@ async function resendIfRateLimited(func: () => Promise, numRetriesAllowed: resendDelay = e.getRetryAfterMs() ?? defaultMs; logger.info(`Rate limited by server, retrying in ${resendDelay}ms`); } catch (e) { - logger.warn(`Error while retrieving a rate-limit retry delay, retrying after default delay of ${defaultMs}`, e); + logger.warn( + `Error while retrieving a rate-limit retry delay, retrying after default delay of ${defaultMs}`, + e, + ); resendDelay = defaultMs; } await sleep(resendDelay); @@ -1183,5 +1195,5 @@ async function resendIfRateLimited(func: () => Promise, numRetriesAllowed: throw e; } } - }; + } } From b96ae12d8d6ad842c958de73b3b88b13d071ce5e Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Wed, 6 Nov 2024 11:40:18 -0500 Subject: [PATCH 3/3] Handle ratelimiting for non-legacy state setting Each request must be retried, as the non-legacy flow involves a sequence of requests that must resolve in order. --- src/matrixrtc/MatrixRTCSession.ts | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/matrixrtc/MatrixRTCSession.ts b/src/matrixrtc/MatrixRTCSession.ts index 992e50695c0..a317495a95d 100644 --- a/src/matrixrtc/MatrixRTCSession.ts +++ b/src/matrixrtc/MatrixRTCSession.ts @@ -1049,11 +1049,8 @@ export class MatrixRTCSession extends TypedEventEmitter + this.client.sendStateEvent(this.room.roomId, EventType.GroupCallMemberPrefix, newContent, stateKey), ); // If sending state cancels your own delayed state, prepare another delayed state // TODO: Remove this once MSC4140 is stable & doesn't cancel own delayed state @@ -1093,11 +1090,13 @@ export class MatrixRTCSession extends TypedEventEmitter + this.client.sendStateEvent( + this.room.roomId, + EventType.GroupCallMemberPrefix, + {}, + this.makeMembershipStateKey(localUserId, localDeviceId), + ), ); } }