diff --git a/CHANGES.md b/CHANGES.md index 88b053897e60..c625e4d56119 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,8 +1,105 @@ -Synapse 1.59.0 -============== +Synapse 1.59.0rc1 (2022-05-10) +============================== + +This release makes several changes that server administrators should be aware of: + +- Device name lookup over federation is now disabled by default. ([\#12616](https://github.com/matrix-org/synapse/issues/12616)) +- The `synapse.app.appservice` and `synapse.app.user_dir` worker application types are now deprecated. ([\#12452](https://github.com/matrix-org/synapse/issues/12452), [\#12654](https://github.com/matrix-org/synapse/issues/12654)) + +See [the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1590) for more details. + +Additionally, this release removes the non-standard `m.login.jwt` login type from Synapse. It can be replaced with `org.matrix.login.jwt` for identical behaviour. This is only used if `jwt_config.enabled` is set to `true` in the configuration. ([\#12597](https://github.com/matrix-org/synapse/issues/12597)) + +Features +-------- + +- Support [MSC3266](https://github.com/matrix-org/matrix-doc/pull/3266) room summaries over federation. ([\#11507](https://github.com/matrix-org/synapse/issues/11507)) +- Implement [changes](https://github.com/matrix-org/matrix-spec-proposals/pull/2285/commits/4a77139249c2e830aec3c7d6bd5501a514d1cc27) to [MSC2285 (hidden read receipts)](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). Contributed by @SimonBrandner. ([\#12168](https://github.com/matrix-org/synapse/issues/12168), [\#12635](https://github.com/matrix-org/synapse/issues/12635), [\#12636](https://github.com/matrix-org/synapse/issues/12636), [\#12670](https://github.com/matrix-org/synapse/issues/12670)) +- Extend the [module API](https://github.com/matrix-org/synapse/blob/release-v1.59/synapse/module_api/__init__.py) to allow modules to change actions for existing push rules of local users. ([\#12406](https://github.com/matrix-org/synapse/issues/12406)) +- Add the `notify_appservices_from_worker` configuration option (superseding `notify_appservices`) to allow a generic worker to be designated as the worker to send traffic to Application Services. ([\#12452](https://github.com/matrix-org/synapse/issues/12452)) +- Add the `update_user_directory_from_worker` configuration option (superseding `update_user_directory`) to allow a generic worker to be designated as the worker to update the user directory. ([\#12654](https://github.com/matrix-org/synapse/issues/12654)) +- Add new `enable_registration_token_3pid_bypass` configuration option to allow registrations via token as an alternative to verifying a 3pid. ([\#12526](https://github.com/matrix-org/synapse/issues/12526)) +- Implement [MSC3786](https://github.com/matrix-org/matrix-spec-proposals/pull/3786): Add a default push rule to ignore `m.room.server_acl` events. ([\#12601](https://github.com/matrix-org/synapse/issues/12601)) +- Add new `mau_appservice_trial_days` configuration option to specify a different trial period for users registered via an appservice. ([\#12619](https://github.com/matrix-org/synapse/issues/12619)) + + +Bugfixes +-------- + +- Fix a bug introduced in Synapse 1.48.0 where the latest thread reply provided failed to include the proper bundled aggregations. ([\#12273](https://github.com/matrix-org/synapse/issues/12273)) +- Fix a bug introduced in Synapse 1.22.0 where attempting to send a large amount of read receipts to an application service all at once would result in duplicate content and abnormally high memory usage. Contributed by Brad & Nick @ Beeper. ([\#12544](https://github.com/matrix-org/synapse/issues/12544)) +- Fix a bug introduced in Synapse 1.57.0 which could cause `Failed to calculate hosts in room` errors to be logged for outbound federation. ([\#12570](https://github.com/matrix-org/synapse/issues/12570)) +- Fix a long-standing bug where status codes would almost always get logged as `200!`, irrespective of the actual status code, when clients disconnect before a request has finished processing. ([\#12580](https://github.com/matrix-org/synapse/issues/12580)) +- Fix race when persisting an event and deleting a room that could lead to outbound federation breaking. ([\#12594](https://github.com/matrix-org/synapse/issues/12594)) +- Fix a bug introduced in Synapse 1.53.0 where bundled aggregations for annotations/edits were incorrectly calculated. ([\#12633](https://github.com/matrix-org/synapse/issues/12633)) +- Fix a long-standing bug where rooms containing power levels with string values could not be upgraded. ([\#12657](https://github.com/matrix-org/synapse/issues/12657)) +- Prevent memory leak from reoccurring when presence is disabled. ([\#12656](https://github.com/matrix-org/synapse/issues/12656)) + + +Updates to the Docker image +--------------------------- + +- Explicitly opt-in to using [BuildKit-specific features](https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md) in the Dockerfile. This fixes issues with building images in some GitLab CI environments. ([\#12541](https://github.com/matrix-org/synapse/issues/12541)) +- Update the "Build docker images" GitHub Actions workflow to use `docker/metadata-action` to generate docker image tags, instead of a custom shell script. Contributed by @henryclw. ([\#12573](https://github.com/matrix-org/synapse/issues/12573)) + + +Improved Documentation +---------------------- + +- Update SQL statements and replace use of old table `user_stats_historical` in docs for Synapse Admins. ([\#12536](https://github.com/matrix-org/synapse/issues/12536)) +- Add missing linebreak to `pipx` install instructions. ([\#12579](https://github.com/matrix-org/synapse/issues/12579)) +- Add information about the TCP replication module to docs. ([\#12621](https://github.com/matrix-org/synapse/issues/12621)) +- Fixes to the formatting of `README.rst`. ([\#12627](https://github.com/matrix-org/synapse/issues/12627)) +- Fix docs on how to run specific Complement tests using the `complement.sh` test runner. ([\#12664](https://github.com/matrix-org/synapse/issues/12664)) -The non-standard `m.login.jwt` login type has been removed from Synapse. It can be replaced with `org.matrix.login.jwt` for identical behaviour. This is only used if `jwt_config.enabled` is set to `true` in the configuration. +Deprecations and Removals +------------------------- + +- Remove unstable identifiers from [MSC3069](https://github.com/matrix-org/matrix-doc/pull/3069). ([\#12596](https://github.com/matrix-org/synapse/issues/12596)) +- Remove the unspecified `m.login.jwt` login type and the unstable `uk.half-shot.msc2778.login.application_service` from + [MSC2778](https://github.com/matrix-org/matrix-doc/pull/2778). ([\#12597](https://github.com/matrix-org/synapse/issues/12597)) +- Synapse now requires at least Python 3.7.1 (up from 3.7.0), for compatibility with the latest Twisted trunk. ([\#12613](https://github.com/matrix-org/synapse/issues/12613)) + + +Internal Changes +---------------- + +- Use supervisord to supervise Postgres and Caddy in the Complement image to reduce restart time. ([\#12480](https://github.com/matrix-org/synapse/issues/12480)) +- Immediately retry any requests that have backed off when a server comes back online. ([\#12500](https://github.com/matrix-org/synapse/issues/12500)) +- Use `make_awaitable` instead of `defer.succeed` for return values of mocks in tests. ([\#12505](https://github.com/matrix-org/synapse/issues/12505)) +- Consistently check if an object is a `frozendict`. ([\#12564](https://github.com/matrix-org/synapse/issues/12564)) +- Protect module callbacks with read semantics against cancellation. ([\#12568](https://github.com/matrix-org/synapse/issues/12568)) +- Improve comments and error messages around access tokens. ([\#12577](https://github.com/matrix-org/synapse/issues/12577)) +- Improve docstrings for the receipts store. ([\#12581](https://github.com/matrix-org/synapse/issues/12581)) +- Use constants for read-receipts in tests. ([\#12582](https://github.com/matrix-org/synapse/issues/12582)) +- Log status code of cancelled requests as 499 and avoid logging stack traces for them. ([\#12587](https://github.com/matrix-org/synapse/issues/12587), [\#12663](https://github.com/matrix-org/synapse/issues/12663)) +- Remove special-case for `twisted` logger from default log config. ([\#12589](https://github.com/matrix-org/synapse/issues/12589)) +- Use `getClientAddress` instead of the deprecated `getClientIP`. ([\#12599](https://github.com/matrix-org/synapse/issues/12599)) +- Add link to documentation in Grafana Dashboard. ([\#12602](https://github.com/matrix-org/synapse/issues/12602)) +- Reduce log spam when running multiple event persisters. ([\#12610](https://github.com/matrix-org/synapse/issues/12610)) +- Add extra debug logging to federation sender. ([\#12614](https://github.com/matrix-org/synapse/issues/12614)) +- Prevent remote homeservers from requesting local user device names by default. ([\#12616](https://github.com/matrix-org/synapse/issues/12616)) +- Add a consistency check on events which we read from the database. ([\#12620](https://github.com/matrix-org/synapse/issues/12620)) +- Remove use of the `constantly` library and switch to enums for `EventRedactBehaviour`. Contributed by @andrewdoh. ([\#12624](https://github.com/matrix-org/synapse/issues/12624)) +- Remove unused code related to receipts. ([\#12632](https://github.com/matrix-org/synapse/issues/12632)) +- Minor improvements to the scripts for running Synapse in worker mode under Complement. ([\#12637](https://github.com/matrix-org/synapse/issues/12637)) +- Move `pympler` back in to the `all` extras. ([\#12652](https://github.com/matrix-org/synapse/issues/12652)) +- Fix spelling of `M_UNRECOGNIZED` in comments. ([\#12665](https://github.com/matrix-org/synapse/issues/12665)) +- Release script: confirm the commit to be tagged before tagging. ([\#12556](https://github.com/matrix-org/synapse/issues/12556)) +- Fix a typo in the announcement text generated by the Synapse release development script. ([\#12612](https://github.com/matrix-org/synapse/issues/12612)) + +### Typechecking + +- Fix scripts-dev to pass typechecking. ([\#12356](https://github.com/matrix-org/synapse/issues/12356)) +- Add some type hints to datastore. ([\#12485](https://github.com/matrix-org/synapse/issues/12485)) +- Remove unused `# type: ignore`s. ([\#12531](https://github.com/matrix-org/synapse/issues/12531)) +- Allow unused `# type: ignore` comments in bleeding edge CI jobs. ([\#12576](https://github.com/matrix-org/synapse/issues/12576)) +- Remove redundant lines of config from `mypy.ini`. ([\#12608](https://github.com/matrix-org/synapse/issues/12608)) +- Update to mypy 0.950. ([\#12650](https://github.com/matrix-org/synapse/issues/12650)) +- Use `Concatenate` to better annotate `_do_execute`. ([\#12666](https://github.com/matrix-org/synapse/issues/12666)) +- Use `ParamSpec` to refine type hints. ([\#12667](https://github.com/matrix-org/synapse/issues/12667)) +- Fix mypy against latest pillow stubs. ([\#12671](https://github.com/matrix-org/synapse/issues/12671)) Synapse 1.58.1 (2022-05-05) =========================== diff --git a/changelog.d/11507.feature b/changelog.d/11507.feature deleted file mode 100644 index 72c5690cca4a..000000000000 --- a/changelog.d/11507.feature +++ /dev/null @@ -1 +0,0 @@ -Support [MSC3266](https://github.com/matrix-org/matrix-doc/pull/3266) room summaries over federation. diff --git a/changelog.d/12168.feature b/changelog.d/12168.feature deleted file mode 100644 index cd5c45029ee1..000000000000 --- a/changelog.d/12168.feature +++ /dev/null @@ -1 +0,0 @@ -Implement [changes](https://github.com/matrix-org/matrix-spec-proposals/pull/2285/commits/4a77139249c2e830aec3c7d6bd5501a514d1cc27) to [MSC2285 (hidden read receipts)](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). Contributed by @SimonBrandner. diff --git a/changelog.d/12273.bugfix b/changelog.d/12273.bugfix deleted file mode 100644 index f8d7b6c88956..000000000000 --- a/changelog.d/12273.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse v1.48.0 where latest thread reply provided failed to include the proper bundled aggregations. diff --git a/changelog.d/12356.misc b/changelog.d/12356.misc deleted file mode 100644 index 43e192910602..000000000000 --- a/changelog.d/12356.misc +++ /dev/null @@ -1 +0,0 @@ -Fix scripts-dev to pass typechecking. \ No newline at end of file diff --git a/changelog.d/12406.feature b/changelog.d/12406.feature deleted file mode 100644 index e345afdee729..000000000000 --- a/changelog.d/12406.feature +++ /dev/null @@ -1 +0,0 @@ -Add a module API to allow modules to change actions for existing push rules of local users. diff --git a/changelog.d/12452.feature b/changelog.d/12452.feature deleted file mode 100644 index 22f054d34493..000000000000 --- a/changelog.d/12452.feature +++ /dev/null @@ -1 +0,0 @@ -Add the `notify_appservices_from_worker` configuration option (superseding `notify_appservices`) to allow a generic worker to be designated as the worker to send traffic to Application Services. \ No newline at end of file diff --git a/changelog.d/12485.misc b/changelog.d/12477.misc similarity index 100% rename from changelog.d/12485.misc rename to changelog.d/12477.misc diff --git a/changelog.d/12480.misc b/changelog.d/12480.misc deleted file mode 100644 index 18a85e7b1579..000000000000 --- a/changelog.d/12480.misc +++ /dev/null @@ -1 +0,0 @@ -Use supervisord to supervise Postgres and Caddy in the Complement image to reduce restart time. \ No newline at end of file diff --git a/changelog.d/12500.misc b/changelog.d/12500.misc deleted file mode 100644 index dbe3f7f5d197..000000000000 --- a/changelog.d/12500.misc +++ /dev/null @@ -1 +0,0 @@ -Immediately retry any requests that have backed off when a server comes back online. diff --git a/changelog.d/12505.misc b/changelog.d/12505.misc deleted file mode 100644 index a691d7962f89..000000000000 --- a/changelog.d/12505.misc +++ /dev/null @@ -1 +0,0 @@ -Use `make_awaitable` instead of `defer.succeed` for return values of mocks in tests. diff --git a/changelog.d/12526.feature b/changelog.d/12526.feature deleted file mode 100644 index c01596282c9f..000000000000 --- a/changelog.d/12526.feature +++ /dev/null @@ -1 +0,0 @@ -Add new `enable_registration_token_3pid_bypass` configuration option to allow registrations via token as an alternative to verifying a 3pid. \ No newline at end of file diff --git a/changelog.d/12531.misc b/changelog.d/12531.misc deleted file mode 100644 index 412fc9b6dc70..000000000000 --- a/changelog.d/12531.misc +++ /dev/null @@ -1 +0,0 @@ -Remove unused `# type: ignore`s. diff --git a/changelog.d/12536.doc b/changelog.d/12536.doc deleted file mode 100644 index 4034c42076c4..000000000000 --- a/changelog.d/12536.doc +++ /dev/null @@ -1 +0,0 @@ -Update SQL statements and replace use of old table `user_stats_historical` in docs for Synapse Admins. diff --git a/changelog.d/12541.docker b/changelog.d/12541.docker deleted file mode 100644 index c3b9c31657cd..000000000000 --- a/changelog.d/12541.docker +++ /dev/null @@ -1 +0,0 @@ -Explicitly opt-in to using [BuildKit-specific features](https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md) in the Dockerfile. This fixes issues with building images in some GitLab CI environments. diff --git a/changelog.d/12544.bugfix b/changelog.d/12544.bugfix deleted file mode 100644 index b5169cd8311a..000000000000 --- a/changelog.d/12544.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug where attempting to send a large amount of read receipts to an application service all at once would result in duplicate content and abnormally high memory usage. Contributed by Brad & Nick @ Beeper. diff --git a/changelog.d/12556.misc b/changelog.d/12556.misc deleted file mode 100644 index dc245397fbcd..000000000000 --- a/changelog.d/12556.misc +++ /dev/null @@ -1 +0,0 @@ -Release script: confirm the commit to be tagged before tagging. diff --git a/changelog.d/12564.misc b/changelog.d/12564.misc deleted file mode 100644 index 207c3224645f..000000000000 --- a/changelog.d/12564.misc +++ /dev/null @@ -1 +0,0 @@ -Consistently check if an object is a `frozendict`. diff --git a/changelog.d/12568.misc b/changelog.d/12568.misc deleted file mode 100644 index f64dc67c4f9a..000000000000 --- a/changelog.d/12568.misc +++ /dev/null @@ -1 +0,0 @@ -Protect module callbacks with read semantics against cancellation. diff --git a/changelog.d/12570.bugfix b/changelog.d/12570.bugfix deleted file mode 100644 index 1038646f358d..000000000000 --- a/changelog.d/12570.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse 1.57 which could cause `Failed to calculate hosts in room` errors to be logged for outbound federation. diff --git a/changelog.d/12573.docker b/changelog.d/12573.docker deleted file mode 100644 index 5cc8de50acb3..000000000000 --- a/changelog.d/12573.docker +++ /dev/null @@ -1 +0,0 @@ -Update the "Build docker images" GitHub Actions workflow to use `docker/metadata-action` to generate docker image tags, instead of a custom shell script. Contributed by henryclw. \ No newline at end of file diff --git a/changelog.d/12576.misc b/changelog.d/12576.misc deleted file mode 100644 index 71022c86337f..000000000000 --- a/changelog.d/12576.misc +++ /dev/null @@ -1 +0,0 @@ -Allow unused `#type: ignore` comments in bleeding edge CI jobs. diff --git a/changelog.d/12577.misc b/changelog.d/12577.misc deleted file mode 100644 index 8c4c47ad5292..000000000000 --- a/changelog.d/12577.misc +++ /dev/null @@ -1 +0,0 @@ -Improve comments and error messages around access tokens. \ No newline at end of file diff --git a/changelog.d/12579.doc b/changelog.d/12579.doc deleted file mode 100644 index bcec5fe1af24..000000000000 --- a/changelog.d/12579.doc +++ /dev/null @@ -1 +0,0 @@ -Add missing linebreak to pipx install instructions. diff --git a/changelog.d/12580.bugfix b/changelog.d/12580.bugfix deleted file mode 100644 index bedce405e2ae..000000000000 --- a/changelog.d/12580.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long standing bug where status codes would almost always get logged as 200!, irrespective of the actual status code, when clients disconnect before a request has finished processing. diff --git a/changelog.d/12581.misc b/changelog.d/12581.misc deleted file mode 100644 index 38d40b262b25..000000000000 --- a/changelog.d/12581.misc +++ /dev/null @@ -1 +0,0 @@ -Improve docstrings for the receipts store. diff --git a/changelog.d/12582.misc b/changelog.d/12582.misc deleted file mode 100644 index 5fa9c9afe86e..000000000000 --- a/changelog.d/12582.misc +++ /dev/null @@ -1 +0,0 @@ -Use constants for read-receipts in tests. diff --git a/changelog.d/12586.misc b/changelog.d/12586.misc new file mode 100644 index 000000000000..d26e332305ce --- /dev/null +++ b/changelog.d/12586.misc @@ -0,0 +1 @@ +Add `@cancellable` decorator, for use on endpoint methods that can be cancelled when clients disconnect. diff --git a/changelog.d/12587.misc b/changelog.d/12587.misc deleted file mode 100644 index 3b466f1ddf16..000000000000 --- a/changelog.d/12587.misc +++ /dev/null @@ -1 +0,0 @@ -Log status code of cancelled requests as 499 and avoid logging stack traces for them. diff --git a/changelog.d/12588.misc b/changelog.d/12588.misc new file mode 100644 index 000000000000..f62d5c8e210c --- /dev/null +++ b/changelog.d/12588.misc @@ -0,0 +1 @@ +Add ability to cancel disconnected requests to `SynapseRequest`. diff --git a/changelog.d/12589.misc b/changelog.d/12589.misc deleted file mode 100644 index d362828d2e59..000000000000 --- a/changelog.d/12589.misc +++ /dev/null @@ -1 +0,0 @@ -Remove special-case for `twisted` logger from default log config. diff --git a/changelog.d/12594.bugfix b/changelog.d/12594.bugfix deleted file mode 100644 index 7411d9c07934..000000000000 --- a/changelog.d/12594.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix race when persisting an event and deleting a room that could lead to outbound federation breaking. diff --git a/changelog.d/12596.removal b/changelog.d/12596.removal deleted file mode 100644 index 14fbfb39540a..000000000000 --- a/changelog.d/12596.removal +++ /dev/null @@ -1 +0,0 @@ -Remove unstable identifiers from [MSC3069](https://github.com/matrix-org/matrix-doc/pull/3069). diff --git a/changelog.d/12597.removal b/changelog.d/12597.removal deleted file mode 100644 index 7927f1d68d5f..000000000000 --- a/changelog.d/12597.removal +++ /dev/null @@ -1,2 +0,0 @@ -Remove the unspecified `m.login.jwt` login type and the unstable `uk.half-shot.msc2778.login.application_service` from -[MSC2778](https://github.com/matrix-org/matrix-doc/pull/2778). diff --git a/changelog.d/12599.misc b/changelog.d/12599.misc deleted file mode 100644 index d01278bbce51..000000000000 --- a/changelog.d/12599.misc +++ /dev/null @@ -1 +0,0 @@ -Use `getClientAddress` instead of the deprecated `getClientIP`. diff --git a/changelog.d/12601.feature b/changelog.d/12601.feature deleted file mode 100644 index c13360ff35c3..000000000000 --- a/changelog.d/12601.feature +++ /dev/null @@ -1 +0,0 @@ -Implement MSC3786: Add a default push rule to ignore m.room.server_acl events. diff --git a/changelog.d/12602.misc b/changelog.d/12602.misc deleted file mode 100644 index cdccc5c31661..000000000000 --- a/changelog.d/12602.misc +++ /dev/null @@ -1 +0,0 @@ -Add link to documentation in Grafana Dashboard. diff --git a/changelog.d/12608.misc b/changelog.d/12608.misc deleted file mode 100644 index 38272118fbe8..000000000000 --- a/changelog.d/12608.misc +++ /dev/null @@ -1 +0,0 @@ -Remove redundant lines of config from `mypy.ini`. \ No newline at end of file diff --git a/changelog.d/12610.misc b/changelog.d/12610.misc deleted file mode 100644 index 02efe0c72f51..000000000000 --- a/changelog.d/12610.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce log spam when running multiple event persisters. diff --git a/changelog.d/12612.bugfix b/changelog.d/12612.bugfix deleted file mode 100644 index c39e97f0cbaa..000000000000 --- a/changelog.d/12612.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a typo in the announcement text generated by the Synapse release development script. \ No newline at end of file diff --git a/changelog.d/12613.removal b/changelog.d/12613.removal deleted file mode 100644 index b1a9e207b00f..000000000000 --- a/changelog.d/12613.removal +++ /dev/null @@ -1 +0,0 @@ -Synapse now requires at least Python 3.7.1 (up from 3.7.0), for compatibility with the latest Twisted trunk. diff --git a/changelog.d/12614.misc b/changelog.d/12614.misc deleted file mode 100644 index 79022df127d0..000000000000 --- a/changelog.d/12614.misc +++ /dev/null @@ -1 +0,0 @@ -Add extra debug logging to federation sender. diff --git a/changelog.d/12616.misc b/changelog.d/12616.misc deleted file mode 100644 index d17ce24cdf29..000000000000 --- a/changelog.d/12616.misc +++ /dev/null @@ -1 +0,0 @@ -Prevent remote homeservers from requesting local user device names by default. \ No newline at end of file diff --git a/changelog.d/12619.feature b/changelog.d/12619.feature deleted file mode 100644 index b0fc0f5fedf8..000000000000 --- a/changelog.d/12619.feature +++ /dev/null @@ -1 +0,0 @@ -Add new `mau_appservice_trial_days` configuration option to specify a different trial period for users registered via an appservice. diff --git a/changelog.d/12620.misc b/changelog.d/12620.misc deleted file mode 100644 index 63f8e540c37c..000000000000 --- a/changelog.d/12620.misc +++ /dev/null @@ -1 +0,0 @@ -Add a consistency check on events which we read from the database. diff --git a/changelog.d/12621.doc b/changelog.d/12621.doc deleted file mode 100644 index d29fb9cb995e..000000000000 --- a/changelog.d/12621.doc +++ /dev/null @@ -1 +0,0 @@ -Add information about the TCP replication module to docs. diff --git a/changelog.d/12624.misc b/changelog.d/12624.misc deleted file mode 100644 index 8772d40fa70f..000000000000 --- a/changelog.d/12624.misc +++ /dev/null @@ -1 +0,0 @@ -Remove use of constantly library and switch to enums for EventRedactBehaviour. Contributed by @andrewdoh. diff --git a/changelog.d/12627.doc b/changelog.d/12627.doc deleted file mode 100644 index 3a787dfef275..000000000000 --- a/changelog.d/12627.doc +++ /dev/null @@ -1 +0,0 @@ -Fixes to the formatting of README.rst. diff --git a/changelog.d/12630.misc b/changelog.d/12630.misc new file mode 100644 index 000000000000..43e12603e2d8 --- /dev/null +++ b/changelog.d/12630.misc @@ -0,0 +1 @@ +Add a helper class for testing request cancellation. diff --git a/changelog.d/12632.misc b/changelog.d/12632.misc deleted file mode 100644 index 9e4ba79c794f..000000000000 --- a/changelog.d/12632.misc +++ /dev/null @@ -1 +0,0 @@ -Remove unused code related to receipts. diff --git a/changelog.d/12633.bugfix b/changelog.d/12633.bugfix deleted file mode 100644 index 32332acd9a36..000000000000 --- a/changelog.d/12633.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse v1.53.0 where bundled aggregations for annotations/edits were incorrectly calculated. diff --git a/changelog.d/12635.feature b/changelog.d/12635.feature deleted file mode 100644 index cd5c45029ee1..000000000000 --- a/changelog.d/12635.feature +++ /dev/null @@ -1 +0,0 @@ -Implement [changes](https://github.com/matrix-org/matrix-spec-proposals/pull/2285/commits/4a77139249c2e830aec3c7d6bd5501a514d1cc27) to [MSC2285 (hidden read receipts)](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). Contributed by @SimonBrandner. diff --git a/changelog.d/12636.feature b/changelog.d/12636.feature deleted file mode 100644 index cd5c45029ee1..000000000000 --- a/changelog.d/12636.feature +++ /dev/null @@ -1 +0,0 @@ -Implement [changes](https://github.com/matrix-org/matrix-spec-proposals/pull/2285/commits/4a77139249c2e830aec3c7d6bd5501a514d1cc27) to [MSC2285 (hidden read receipts)](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). Contributed by @SimonBrandner. diff --git a/changelog.d/12637.misc b/changelog.d/12637.misc deleted file mode 100644 index 735257787fed..000000000000 --- a/changelog.d/12637.misc +++ /dev/null @@ -1 +0,0 @@ -Minor improvements to the scripts for running Synapse in worker mode under Complement. diff --git a/changelog.d/12639.bugfix b/changelog.d/12639.bugfix deleted file mode 100644 index c01596282c9f..000000000000 --- a/changelog.d/12639.bugfix +++ /dev/null @@ -1 +0,0 @@ -Add new `enable_registration_token_3pid_bypass` configuration option to allow registrations via token as an alternative to verifying a 3pid. \ No newline at end of file diff --git a/changelog.d/12650.misc b/changelog.d/12650.misc deleted file mode 100644 index 07bb4ce5a91a..000000000000 --- a/changelog.d/12650.misc +++ /dev/null @@ -1 +0,0 @@ -Update to mypy 0.950. \ No newline at end of file diff --git a/changelog.d/12652.misc b/changelog.d/12652.misc deleted file mode 100644 index 7b7c1cf5ff2a..000000000000 --- a/changelog.d/12652.misc +++ /dev/null @@ -1 +0,0 @@ -Move `pympler` back in to the `all` extras. diff --git a/changelog.d/12656.misc b/changelog.d/12656.misc deleted file mode 100644 index 8a8743e614f3..000000000000 --- a/changelog.d/12656.misc +++ /dev/null @@ -1 +0,0 @@ -Prevent memory leak from reoccurring when presence is disabled. diff --git a/changelog.d/12657.bugfix b/changelog.d/12657.bugfix deleted file mode 100644 index 7547ca40a724..000000000000 --- a/changelog.d/12657.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug where rooms containing power levels with string values could not be upgraded. diff --git a/changelog.d/12663.misc b/changelog.d/12663.misc deleted file mode 100644 index 3b466f1ddf16..000000000000 --- a/changelog.d/12663.misc +++ /dev/null @@ -1 +0,0 @@ -Log status code of cancelled requests as 499 and avoid logging stack traces for them. diff --git a/changelog.d/12664.doc b/changelog.d/12664.doc deleted file mode 100644 index 142d18037a12..000000000000 --- a/changelog.d/12664.doc +++ /dev/null @@ -1 +0,0 @@ -Fix docs on how to run specific Complement tests using the `complement.sh` test runner. diff --git a/changelog.d/12665.misc b/changelog.d/12665.misc deleted file mode 100644 index 37b96fea37d3..000000000000 --- a/changelog.d/12665.misc +++ /dev/null @@ -1 +0,0 @@ -Fix spelling of `M_UNRECOGNIZED` in comments. diff --git a/changelog.d/12666.misc b/changelog.d/12666.misc deleted file mode 100644 index 96268e33f56f..000000000000 --- a/changelog.d/12666.misc +++ /dev/null @@ -1 +0,0 @@ -Use `Concatenate` to better annotate `_do_execute`. diff --git a/changelog.d/12667.misc b/changelog.d/12667.misc deleted file mode 100644 index 2b17502d6b57..000000000000 --- a/changelog.d/12667.misc +++ /dev/null @@ -1 +0,0 @@ -Use `ParamSpec` to refine type hints. diff --git a/changelog.d/12670.feature b/changelog.d/12670.feature deleted file mode 100644 index cd5c45029ee1..000000000000 --- a/changelog.d/12670.feature +++ /dev/null @@ -1 +0,0 @@ -Implement [changes](https://github.com/matrix-org/matrix-spec-proposals/pull/2285/commits/4a77139249c2e830aec3c7d6bd5501a514d1cc27) to [MSC2285 (hidden read receipts)](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). Contributed by @SimonBrandner. diff --git a/changelog.d/12671.misc b/changelog.d/12671.misc deleted file mode 100644 index 56df4e383154..000000000000 --- a/changelog.d/12671.misc +++ /dev/null @@ -1 +0,0 @@ -Fix mypy against latest pillow stubs. diff --git a/changelog.d/12679.misc b/changelog.d/12679.misc new file mode 100644 index 000000000000..6df1116b49ee --- /dev/null +++ b/changelog.d/12679.misc @@ -0,0 +1 @@ +Preparation for database schema simplifications: stop writing to `event_reference_hashes`. diff --git a/changelog.d/12689.misc b/changelog.d/12689.misc new file mode 100644 index 000000000000..daa484ea3019 --- /dev/null +++ b/changelog.d/12689.misc @@ -0,0 +1 @@ +Refactor `EventContext` class. diff --git a/changelog.d/12694.misc b/changelog.d/12694.misc new file mode 100644 index 000000000000..e1e956a51301 --- /dev/null +++ b/changelog.d/12694.misc @@ -0,0 +1 @@ +Capture the `Deferred` for request cancellation in `_AsyncResource`. diff --git a/changelog.d/12695.misc b/changelog.d/12695.misc new file mode 100644 index 000000000000..1b39d969a4c5 --- /dev/null +++ b/changelog.d/12695.misc @@ -0,0 +1 @@ +Fixes an incorrect type hint for `Filter._check_event_relations`. diff --git a/debian/changelog b/debian/changelog index 5b21e0d369e0..fabc690bae93 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,10 +1,11 @@ -matrix-synapse-py3 (1.58.2) UNRELEASED; urgency=medium +matrix-synapse-py3 (1.59.0~rc1) stable; urgency=medium * Adjust how the `exported-requirements.txt` file is generated as part of the process of building these packages. This affects the package maintainers only; end-users are unaffected. + * New Synapse release 1.59.0rc1. - -- Synapse Packaging team Fri, 06 May 2022 13:49:29 +0100 + -- Synapse Packaging team Tue, 10 May 2022 10:45:08 +0100 matrix-synapse-py3 (1.58.1) stable; urgency=medium diff --git a/docs/upgrade.md b/docs/upgrade.md index 18c33a4198ab..fa4b3ef5902d 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -101,29 +101,36 @@ To re-enable this functionality, set the homeserver config option to `true`. -## Deprecation of the `synapse.app.appservice` worker application type +## Deprecation of the `synapse.app.appservice` and `synapse.app.user_dir` worker application types The `synapse.app.appservice` worker application type allowed you to configure a single worker to use to notify application services of new events, as long as this functionality was disabled on the main process with `notify_appservices: False`. +Further, the `synapse.app.user_dir` worker application type allowed you to configure +a single worker to be responsible for updating the user directory, as long as this +was disabled on the main process with `update_user_directory: False`. To unify Synapse's worker types, the `synapse.app.appservice` worker application type and the `notify_appservices` configuration option have been deprecated. +The `synapse.app.user_dir` worker application type and `update_user_directory` +configuration option have also been deprecated. -To get the same functionality, it's now recommended that the `synapse.app.generic_worker` -worker application type is used and that the `notify_appservices_from_worker` option -is set to the name of a worker. +To get the same functionality as was provided by the deprecated options, it's now recommended that the `synapse.app.generic_worker` +worker application type is used and that the `notify_appservices_from_worker` and/or +`update_user_directory_from_worker` options are set to the name of a worker. -For the time being, `notify_appservices_from_worker` can be used alongside -`synapse.app.appservice` and `notify_appservices` to make it easier to transition -between the two configurations, however please note that: +For the time being, the old options can be used alongside the new options to make +it easier to transition between the two configurations, however please note that: - the options must not contradict each other (otherwise Synapse won't start); and -- the `notify_appservices` option will be removed in a future release of Synapse. +- the `notify_appservices` and `update_user_directory` options will be removed in a future release of Synapse. -Please see [the relevant section of the worker documentation][v1_59_notify_ases_from] for more information. +Please see the [*Notifying Application Services*][v1_59_notify_ases_from] and +[*Updating the User Directory*][v1_59_update_user_dir] sections of the worker +documentation for more information. [v1_59_notify_ases_from]: workers.md#notifying-application-services +[v1_59_update_user_dir]: workers.md#updating-the-user-directory # Upgrading to v1.58.0 diff --git a/docs/workers.md b/docs/workers.md index 1d049b6c4f28..553792d2384c 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -426,7 +426,7 @@ the shared configuration would include: run_background_tasks_on: background_worker ``` -You might also wish to investigate the `update_user_directory` and +You might also wish to investigate the `update_user_directory_from_worker` and `media_instance_running_background_jobs` settings. An example for a dedicated background worker instance: @@ -435,9 +435,26 @@ An example for a dedicated background worker instance: {{#include systemd-with-workers/workers/background_worker.yaml}} ``` +#### Updating the User Directory + +You can designate one generic worker to update the user directory. + +Specify its name in the shared configuration as follows: + +```yaml +update_user_directory_from_worker: worker_name +``` + +This work cannot be load-balanced; please ensure the main process is restarted +after setting this option in the shared configuration! + +This style of configuration supersedes the legacy `synapse.app.user_dir` +worker application type. + + #### Notifying Application Services -You can designate one worker to send output traffic to Application Services. +You can designate one generic worker to send output traffic to Application Services. Specify its name in the shared configuration as follows: @@ -470,7 +487,7 @@ pusher_instances: ### `synapse.app.appservice` -**Deprecated as of Synapse v1.58.** [Use `synapse.app.generic_worker` with the +**Deprecated as of Synapse v1.59.** [Use `synapse.app.generic_worker` with the `notify_appservices_from_worker` option instead.](#notifying-application-services) Handles sending output traffic to Application Services. Doesn't handle any @@ -540,6 +557,9 @@ Note that if a reverse proxy is used , then `/_matrix/media/` must be routed for ### `synapse.app.user_dir` +**Deprecated as of Synapse v1.59.** [Use `synapse.app.generic_worker` with the +`update_user_directory_from_worker` option instead.](#updating-the-user-directory) + Handles searches in the user directory. It can handle REST endpoints matching the following regular expressions: diff --git a/pyproject.toml b/pyproject.toml index 2c4b7eb08ecd..e3d81ae5f1e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ skip_gitignore = true [tool.poetry] name = "matrix-synapse" -version = "1.58.1" +version = "1.59.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "Apache-2.0" diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 4a808e33fee1..b91ce06de7c3 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -19,6 +19,7 @@ TYPE_CHECKING, Awaitable, Callable, + Collection, Dict, Iterable, List, @@ -444,9 +445,9 @@ def filter_rooms(self, room_ids: Iterable[str]) -> Set[str]: return room_ids async def _check_event_relations( - self, events: Iterable[FilterEvent] + self, events: Collection[FilterEvent] ) -> List[FilterEvent]: - # The event IDs to check, mypy doesn't understand the ifinstance check. + # The event IDs to check, mypy doesn't understand the isinstance check. event_ids = [event.event_id for event in events if isinstance(event, EventBase)] # type: ignore[attr-defined] event_ids_to_keep = set( await self._store.events_have_relations( diff --git a/synapse/app/admin_cmd.py b/synapse/app/admin_cmd.py index 2b0d92cbaedc..2a4c2e59cda3 100644 --- a/synapse/app/admin_cmd.py +++ b/synapse/app/admin_cmd.py @@ -210,7 +210,7 @@ def start(config_options: List[str]) -> None: config.logging.no_redirect_stdio = True # Explicitly disable background processes - config.server.update_user_directory = False + config.worker.should_update_user_directory = False config.worker.run_background_tasks = False config.worker.start_pushers = False config.worker.pusher_shard_config.instances = [] diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index 07dddc0b1326..2a9480a5c161 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -441,22 +441,6 @@ def start(config_options: List[str]) -> None: "synapse.app.user_dir", ) - if config.worker.worker_app == "synapse.app.user_dir": - if config.server.update_user_directory: - sys.stderr.write( - "\nThe update_user_directory must be disabled in the main synapse process" - "\nbefore they can be run in a separate worker." - "\nPlease add ``update_user_directory: false`` to the main config" - "\n" - ) - sys.exit(1) - - # Force the pushers to start since they will be disabled in the main config - config.server.update_user_directory = True - else: - # For other worker types we force this to off. - config.server.update_user_directory = False - synapse.events.USE_FROZEN_DICTS = config.server.use_frozen_dicts synapse.util.caches.TRACK_MEMORY_USAGE = config.caches.track_memory_usage diff --git a/synapse/config/server.py b/synapse/config/server.py index 1e709c7cf519..005a3ee48ce4 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -319,10 +319,6 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: self.presence_router_config, ) = load_module(presence_router_config, ("presence", "presence_router")) - # Whether to update the user directory or not. This should be set to - # false only if we are updating the user directory in a worker - self.update_user_directory = config.get("update_user_directory", True) - # whether to enable the media repository endpoints. This should be set # to false if the media repository is running as a separate endpoint; # doing so ensures that we will not run cache cleanup jobs on the diff --git a/synapse/config/workers.py b/synapse/config/workers.py index a9dbcc6d3dc7..e1569b3c14f9 100644 --- a/synapse/config/workers.py +++ b/synapse/config/workers.py @@ -311,6 +311,13 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: new_option_name="notify_appservices_from_worker", ) + self.should_update_user_directory = self._should_this_worker_perform_duty( + config, + legacy_master_option_name="update_user_directory", + legacy_worker_app_name="synapse.app.user_dir", + new_option_name="update_user_directory_from_worker", + ) + def _should_this_worker_perform_duty( self, config: Dict[str, Any], diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py index 46042b2bf7af..9ccd24b298bb 100644 --- a/synapse/events/snapshot.py +++ b/synapse/events/snapshot.py @@ -15,12 +15,10 @@ import attr from frozendict import frozendict - -from twisted.internet.defer import Deferred +from typing_extensions import Literal from synapse.appservice import ApplicationService from synapse.events import EventBase -from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.types import JsonDict, StateMap if TYPE_CHECKING: @@ -60,6 +58,9 @@ class EventContext: If ``state_group`` is None (ie, the event is an outlier), ``state_group_before_event`` will always also be ``None``. + state_delta_due_to_event: If `state_group` and `state_group_before_event` are not None + then this is the delta of the state between the two groups. + prev_group: If it is known, ``state_group``'s prev_group. Note that this being None does not necessarily mean that ``state_group`` does not have a prev_group! @@ -78,73 +79,47 @@ class EventContext: app_service: If this event is being sent by a (local) application service, that app service. - _current_state_ids: The room state map, including this event - ie, the state - in ``state_group``. - - (type, state_key) -> event_id - - For an outlier, this is {} - - Note that this is a private attribute: it should be accessed via - ``get_current_state_ids``. _AsyncEventContext impl calculates this - on-demand: it will be None until that happens. - - _prev_state_ids: The room state map, excluding this event - ie, the state - in ``state_group_before_event``. For a non-state - event, this will be the same as _current_state_events. - - Note that it is a completely different thing to prev_group! - - (type, state_key) -> event_id - - For an outlier, this is {} - - As with _current_state_ids, this is a private attribute. It should be - accessed via get_prev_state_ids. - partial_state: if True, we may be storing this event with a temporary, incomplete state. """ - rejected: Union[bool, str] = False + _storage: "Storage" + rejected: Union[Literal[False], str] = False _state_group: Optional[int] = None state_group_before_event: Optional[int] = None + _state_delta_due_to_event: Optional[StateMap[str]] = None prev_group: Optional[int] = None delta_ids: Optional[StateMap[str]] = None app_service: Optional[ApplicationService] = None - _current_state_ids: Optional[StateMap[str]] = None - _prev_state_ids: Optional[StateMap[str]] = None - partial_state: bool = False @staticmethod def with_state( + storage: "Storage", state_group: Optional[int], state_group_before_event: Optional[int], - current_state_ids: Optional[StateMap[str]], - prev_state_ids: Optional[StateMap[str]], + state_delta_due_to_event: Optional[StateMap[str]], partial_state: bool, prev_group: Optional[int] = None, delta_ids: Optional[StateMap[str]] = None, ) -> "EventContext": return EventContext( - current_state_ids=current_state_ids, - prev_state_ids=prev_state_ids, + storage=storage, state_group=state_group, state_group_before_event=state_group_before_event, + state_delta_due_to_event=state_delta_due_to_event, prev_group=prev_group, delta_ids=delta_ids, partial_state=partial_state, ) @staticmethod - def for_outlier() -> "EventContext": + def for_outlier( + storage: "Storage", + ) -> "EventContext": """Return an EventContext instance suitable for persisting an outlier event""" - return EventContext( - current_state_ids={}, - prev_state_ids={}, - ) + return EventContext(storage=storage) async def serialize(self, event: EventBase, store: "DataStore") -> JsonDict: """Converts self to a type that can be serialized as JSON, and then @@ -157,24 +132,14 @@ async def serialize(self, event: EventBase, store: "DataStore") -> JsonDict: The serialized event. """ - # We don't serialize the full state dicts, instead they get pulled out - # of the DB on the other side. However, the other side can't figure out - # the prev_state_ids, so if we're a state event we include the event - # id that we replaced in the state. - if event.is_state(): - prev_state_ids = await self.get_prev_state_ids() - prev_state_id = prev_state_ids.get((event.type, event.state_key)) - else: - prev_state_id = None - return { - "prev_state_id": prev_state_id, - "event_type": event.type, - "event_state_key": event.get_state_key(), "state_group": self._state_group, "state_group_before_event": self.state_group_before_event, "rejected": self.rejected, "prev_group": self.prev_group, + "state_delta_due_to_event": _encode_state_dict( + self._state_delta_due_to_event + ), "delta_ids": _encode_state_dict(self.delta_ids), "app_service_id": self.app_service.id if self.app_service else None, "partial_state": self.partial_state, @@ -192,16 +157,16 @@ def deserialize(storage: "Storage", input: JsonDict) -> "EventContext": Returns: The event context. """ - context = _AsyncEventContextImpl( + context = EventContext( # We use the state_group and prev_state_id stuff to pull the # current_state_ids out of the DB and construct prev_state_ids. storage=storage, - prev_state_id=input["prev_state_id"], - event_type=input["event_type"], - event_state_key=input["event_state_key"], state_group=input["state_group"], state_group_before_event=input["state_group_before_event"], prev_group=input["prev_group"], + state_delta_due_to_event=_decode_state_dict( + input["state_delta_due_to_event"] + ), delta_ids=_decode_state_dict(input["delta_ids"]), rejected=input["rejected"], partial_state=input.get("partial_state", False), @@ -249,8 +214,15 @@ async def get_current_state_ids(self) -> Optional[StateMap[str]]: if self.rejected: raise RuntimeError("Attempt to access state_ids of rejected event") - await self._ensure_fetched() - return self._current_state_ids + assert self._state_delta_due_to_event is not None + + prev_state_ids = await self.get_prev_state_ids() + + if self._state_delta_due_to_event: + prev_state_ids = dict(prev_state_ids) + prev_state_ids.update(self._state_delta_due_to_event) + + return prev_state_ids async def get_prev_state_ids(self) -> StateMap[str]: """ @@ -265,94 +237,10 @@ async def get_prev_state_ids(self) -> StateMap[str]: Maps a (type, state_key) to the event ID of the state event matching this tuple. """ - await self._ensure_fetched() - # There *should* be previous state IDs now. - assert self._prev_state_ids is not None - return self._prev_state_ids - - def get_cached_current_state_ids(self) -> Optional[StateMap[str]]: - """Gets the current state IDs if we have them already cached. - - It is an error to access this for a rejected event, since rejected state should - not make it into the room state. This method will raise an exception if - ``rejected`` is set. - - Returns: - Returns None if we haven't cached the state or if state_group is None - (which happens when the associated event is an outlier). - - Otherwise, returns the the current state IDs. - """ - if self.rejected: - raise RuntimeError("Attempt to access state_ids of rejected event") - - return self._current_state_ids - - async def _ensure_fetched(self) -> None: - return None - - -@attr.s(slots=True) -class _AsyncEventContextImpl(EventContext): - """ - An implementation of EventContext which fetches _current_state_ids and - _prev_state_ids from the database on demand. - - Attributes: - - _storage - - _fetching_state_deferred: Resolves when *_state_ids have been calculated. - None if we haven't started calculating yet - - _event_type: The type of the event the context is associated with. - - _event_state_key: The state_key of the event the context is associated with. - - _prev_state_id: If the event associated with the context is a state event, - then `_prev_state_id` is the event_id of the state that was replaced. - """ - - # This needs to have a default as we're inheriting - _storage: "Storage" = attr.ib(default=None) - _prev_state_id: Optional[str] = attr.ib(default=None) - _event_type: str = attr.ib(default=None) - _event_state_key: Optional[str] = attr.ib(default=None) - _fetching_state_deferred: Optional["Deferred[None]"] = attr.ib(default=None) - - async def _ensure_fetched(self) -> None: - if not self._fetching_state_deferred: - self._fetching_state_deferred = run_in_background(self._fill_out_state) - - await make_deferred_yieldable(self._fetching_state_deferred) - - async def _fill_out_state(self) -> None: - """Called to populate the _current_state_ids and _prev_state_ids - attributes by loading from the database. - """ - if self.state_group is None: - # No state group means the event is an outlier. Usually the state_ids dicts are also - # pre-set to empty dicts, but they get reset when the context is serialized, so set - # them to empty dicts again here. - self._current_state_ids = {} - self._prev_state_ids = {} - return - - current_state_ids = await self._storage.state.get_state_ids_for_group( - self.state_group + assert self.state_group_before_event is not None + return await self._storage.state.get_state_ids_for_group( + self.state_group_before_event ) - # Set this separately so mypy knows current_state_ids is not None. - self._current_state_ids = current_state_ids - if self._event_state_key is not None: - self._prev_state_ids = dict(current_state_ids) - - key = (self._event_type, self._event_state_key) - if self._prev_state_id: - self._prev_state_ids[key] = self._prev_state_id - else: - self._prev_state_ids.pop(key, None) - else: - self._prev_state_ids = current_state_ids def _encode_state_dict( diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 38dc5b1f6edf..be5099b507f6 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -659,7 +659,7 @@ async def do_knock( # in the invitee's sync stream. It is stripped out for all other local users. event.unsigned["knock_room_state"] = stripped_room_state["knock_state_events"] - context = EventContext.for_outlier() + context = EventContext.for_outlier(self.storage) stream_id = await self._federation_event_handler.persist_events_and_notify( event.room_id, [(event, context)] ) @@ -848,7 +848,7 @@ async def on_invite_request( ) ) - context = EventContext.for_outlier() + context = EventContext.for_outlier(self.storage) await self._federation_event_handler.persist_events_and_notify( event.room_id, [(event, context)] ) @@ -877,7 +877,7 @@ async def do_remotely_reject_invite( await self.federation_client.send_leave(host_list, event) - context = EventContext.for_outlier() + context = EventContext.for_outlier(self.storage) stream_id = await self._federation_event_handler.persist_events_and_notify( event.room_id, [(event, context)] ) diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index 4492ccf9c685..761caa04b726 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -1423,7 +1423,7 @@ def prep(event: EventBase) -> Optional[Tuple[EventBase, EventContext]]: # we're not bothering about room state, so flag the event as an outlier. event.internal_metadata.outlier = True - context = EventContext.for_outlier() + context = EventContext.for_outlier(self._storage) try: validate_event_for_room_version(room_version_obj, event) check_auth_rules_for_event(room_version_obj, event, auth) @@ -1874,10 +1874,10 @@ async def _update_context_for_auth_events( ) return EventContext.with_state( + storage=self._storage, state_group=state_group, state_group_before_event=context.state_group_before_event, - current_state_ids=current_state_ids, - prev_state_ids=prev_state_ids, + state_delta_due_to_event=state_updates, prev_group=prev_group, delta_ids=state_updates, partial_state=context.partial_state, diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 53b839e2c225..4a4b535bae6a 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -757,6 +757,10 @@ async def deduplicate_state_event( The previous version of the event is returned, if it is found in the event context. Otherwise, None is returned. """ + if event.internal_metadata.is_outlier(): + # This can happen due to out of band memberships + return None + prev_state_ids = await context.get_prev_state_ids() prev_event_id = prev_state_ids.get((event.type, event.state_key)) if not prev_event_id: @@ -1001,7 +1005,7 @@ async def create_new_client_event( # after it is created if builder.internal_metadata.outlier: event.internal_metadata.outlier = True - context = EventContext.for_outlier() + context = EventContext.for_outlier(self.storage) elif ( event.type == EventTypes.MSC2716_INSERTION and state_event_ids diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 048fd4bb8225..74f7fdfe6ce5 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -60,7 +60,7 @@ def __init__(self, hs: "HomeServer"): self.clock = hs.get_clock() self.notifier = hs.get_notifier() self.is_mine_id = hs.is_mine_id - self.update_user_directory = hs.config.server.update_user_directory + self.update_user_directory = hs.config.worker.should_update_user_directory self.search_all_users = hs.config.userdirectory.user_directory_search_all_users self.spam_checker = hs.get_spam_checker() # The current position in the current_state_delta stream diff --git a/synapse/http/server.py b/synapse/http/server.py index 657bffcddd88..4b4debc5cd2b 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -33,6 +33,7 @@ Optional, Pattern, Tuple, + TypeVar, Union, ) @@ -92,6 +93,66 @@ HTTP_STATUS_REQUEST_CANCELLED = 499 +F = TypeVar("F", bound=Callable[..., Any]) + + +_cancellable_method_names = frozenset( + { + # `RestServlet`, `BaseFederationServlet` and `BaseFederationServerServlet` + # methods + "on_GET", + "on_PUT", + "on_POST", + "on_DELETE", + # `_AsyncResource`, `DirectServeHtmlResource` and `DirectServeJsonResource` + # methods + "_async_render_GET", + "_async_render_PUT", + "_async_render_POST", + "_async_render_DELETE", + "_async_render_OPTIONS", + # `ReplicationEndpoint` methods + "_handle_request", + } +) + + +def cancellable(method: F) -> F: + """Marks a servlet method as cancellable. + + Methods with this decorator will be cancelled if the client disconnects before we + finish processing the request. + + During cancellation, `Deferred.cancel()` will be invoked on the `Deferred` wrapping + the method. The `cancel()` call will propagate down to the `Deferred` that is + currently being waited on. That `Deferred` will raise a `CancelledError`, which will + propagate up, as per normal exception handling. + + Before applying this decorator to a new endpoint, you MUST recursively check + that all `await`s in the function are on `async` functions or `Deferred`s that + handle cancellation cleanly, otherwise a variety of bugs may occur, ranging from + premature logging context closure, to stuck requests, to database corruption. + + Usage: + class SomeServlet(RestServlet): + @cancellable + async def on_GET(self, request: SynapseRequest) -> ...: + ... + """ + if method.__name__ not in _cancellable_method_names: + raise ValueError( + "@cancellable decorator can only be applied to servlet methods." + ) + + method.cancellable = True # type: ignore[attr-defined] + return method + + +def is_method_cancellable(method: Callable[..., Any]) -> bool: + """Checks whether a servlet method has the `@cancellable` flag.""" + return getattr(method, "cancellable", False) + + def return_json_error(f: failure.Failure, request: SynapseRequest) -> None: """Sends a JSON error response to clients.""" @@ -283,7 +344,9 @@ def __init__(self, extract_context: bool = False): def render(self, request: SynapseRequest) -> int: """This gets called by twisted every time someone sends us a request.""" - defer.ensureDeferred(self._async_render_wrapper(request)) + request.render_deferred = defer.ensureDeferred( + self._async_render_wrapper(request) + ) return NOT_DONE_YET @wrap_async_request_handler diff --git a/synapse/http/site.py b/synapse/http/site.py index 0b85a57d7787..eeec74b78ae5 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -19,6 +19,7 @@ import attr from zope.interface import implementer +from twisted.internet.defer import Deferred from twisted.internet.interfaces import IAddress, IReactorTime from twisted.python.failure import Failure from twisted.web.http import HTTPChannel @@ -91,6 +92,14 @@ def __init__( # we can't yet create the logcontext, as we don't know the method. self.logcontext: Optional[LoggingContext] = None + # The `Deferred` to cancel if the client disconnects early and + # `is_render_cancellable` is set. Expected to be set by `Resource.render`. + self.render_deferred: Optional["Deferred[None]"] = None + # A boolean indicating whether `render_deferred` should be cancelled if the + # client disconnects early. Expected to be set by the coroutine started by + # `Resource.render`, if rendering is asynchronous. + self.is_render_cancellable = False + global _next_request_seq self.request_seq = _next_request_seq _next_request_seq += 1 @@ -357,7 +366,21 @@ def connectionLost(self, reason: Union[Failure, Exception]) -> None: {"event": "client connection lost", "reason": str(reason.value)} ) - if not self._is_processing: + if self._is_processing: + if self.is_render_cancellable: + if self.render_deferred is not None: + # Throw a cancellation into the request processing, in the hope + # that it will finish up sooner than it normally would. + # The `self.processing()` context manager will call + # `_finished_processing()` when done. + with PreserveLoggingContext(): + self.render_deferred.cancel() + else: + logger.error( + "Connection from client lost, but have no Deferred to " + "cancel even though the request is marked as cancellable." + ) + else: self._finished_processing() def _started_processing(self, servlet_name: str) -> None: diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index cad3b4264007..54e41d537584 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -130,6 +130,7 @@ def __init__(self, hs: "HomeServer"): self.state_store = hs.get_storage().state self.hs = hs self._state_resolution_handler = hs.get_state_resolution_handler() + self._storage = hs.get_storage() @overload async def get_current_state( @@ -361,10 +362,10 @@ async def compute_event_context( if not event.is_state(): return EventContext.with_state( + storage=self._storage, state_group_before_event=state_group_before_event, state_group=state_group_before_event, - current_state_ids=state_ids_before_event, - prev_state_ids=state_ids_before_event, + state_delta_due_to_event={}, prev_group=state_group_before_event_prev_group, delta_ids=deltas_to_state_group_before_event, partial_state=partial_state, @@ -393,10 +394,10 @@ async def compute_event_context( ) return EventContext.with_state( + storage=self._storage, state_group=state_group_after_event, state_group_before_event=state_group_before_event, - current_state_ids=state_ids_after_event, - prev_state_ids=state_ids_before_event, + state_delta_due_to_event=delta_ids, prev_group=state_group_before_event, delta_ids=delta_ids, partial_state=partial_state, diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index ed29a0a5e2db..f544bcfff07f 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -36,7 +36,6 @@ import synapse.metrics from synapse.api.constants import EventContentFields, EventTypes, RelationTypes from synapse.api.room_versions import RoomVersions -from synapse.crypto.event_signing import compute_event_reference_hash from synapse.events import EventBase # noqa: F401 from synapse.events.snapshot import EventContext # noqa: F401 from synapse.storage._base import db_to_json, make_in_list_sql_clause @@ -50,7 +49,7 @@ from synapse.storage.engines.postgres import PostgresEngine from synapse.storage.util.id_generators import AbstractStreamIdGenerator from synapse.storage.util.sequence import SequenceGenerator -from synapse.types import StateMap, get_domain_from_id +from synapse.types import JsonDict, StateMap, get_domain_from_id from synapse.util import json_encoder from synapse.util.iterutils import batch_iter, sorted_topologically @@ -129,7 +128,6 @@ async def _persist_events_and_state_updates( self, events_and_contexts: List[Tuple[EventBase, EventContext]], *, - current_state_for_room: Dict[str, StateMap[str]], state_delta_for_room: Dict[str, DeltaState], new_forward_extremities: Dict[str, Set[str]], use_negative_stream_ordering: bool = False, @@ -140,8 +138,6 @@ async def _persist_events_and_state_updates( Args: events_and_contexts: - current_state_for_room: Map from room_id to the current state of - the room based on forward extremities state_delta_for_room: Map from room_id to the delta to apply to room state new_forward_extremities: Map from room_id to set of event IDs @@ -216,9 +212,6 @@ async def _persist_events_and_state_updates( event_counter.labels(event.type, origin_type, origin_entity).inc() - for room_id, new_state in current_state_for_room.items(): - self.store.get_current_state_ids.prefill((room_id,), new_state) - for room_id, latest_event_ids in new_forward_extremities.items(): self.store.get_latest_event_ids_in_room.prefill( (room_id,), list(latest_event_ids) @@ -236,7 +229,9 @@ async def _get_events_which_are_prevs(self, event_ids: Iterable[str]) -> List[st """ results: List[str] = [] - def _get_events_which_are_prevs_txn(txn, batch): + def _get_events_which_are_prevs_txn( + txn: LoggingTransaction, batch: Collection[str] + ) -> None: sql = """ SELECT prev_event_id, internal_metadata FROM event_edges @@ -286,7 +281,9 @@ async def _get_prevs_before_rejected(self, event_ids: Iterable[str]) -> Set[str] # and their prev events. existing_prevs = set() - def _get_prevs_before_rejected_txn(txn, batch): + def _get_prevs_before_rejected_txn( + txn: LoggingTransaction, batch: Collection[str] + ) -> None: to_recursively_check = batch while to_recursively_check: @@ -516,7 +513,7 @@ def _persist_event_auth_chain_txn( @classmethod def _add_chain_cover_index( cls, - txn, + txn: LoggingTransaction, db_pool: DatabasePool, event_chain_id_gen: SequenceGenerator, event_to_room_id: Dict[str, str], @@ -810,7 +807,7 @@ def _add_chain_cover_index( @staticmethod def _allocate_chain_ids( - txn, + txn: LoggingTransaction, db_pool: DatabasePool, event_chain_id_gen: SequenceGenerator, event_to_room_id: Dict[str, str], @@ -944,7 +941,7 @@ def _persist_transaction_ids_txn( self, txn: LoggingTransaction, events_and_contexts: List[Tuple[EventBase, EventContext]], - ): + ) -> None: """Persist the mapping from transaction IDs to event IDs (if defined).""" to_insert = [] @@ -998,7 +995,7 @@ def _update_current_state_txn( txn: LoggingTransaction, state_delta_by_room: Dict[str, DeltaState], stream_id: int, - ): + ) -> None: for room_id, delta_state in state_delta_by_room.items(): to_delete = delta_state.to_delete to_insert = delta_state.to_insert @@ -1156,7 +1153,7 @@ def _update_current_state_txn( txn, room_id, members_changed ) - def _upsert_room_version_txn(self, txn: LoggingTransaction, room_id: str): + def _upsert_room_version_txn(self, txn: LoggingTransaction, room_id: str) -> None: """Update the room version in the database based off current state events. @@ -1190,7 +1187,7 @@ def _update_forward_extremities_txn( txn: LoggingTransaction, new_forward_extremities: Dict[str, Set[str]], max_stream_order: int, - ): + ) -> None: for room_id in new_forward_extremities.keys(): self.db_pool.simple_delete_txn( txn, table="event_forward_extremities", keyvalues={"room_id": room_id} @@ -1255,9 +1252,9 @@ def _filter_events_and_contexts_for_duplicates( def _update_room_depths_txn( self, - txn, + txn: LoggingTransaction, events_and_contexts: List[Tuple[EventBase, EventContext]], - ): + ) -> None: """Update min_depth for each room Args: @@ -1386,7 +1383,7 @@ def _store_event_txn( # nothing to do here return - def event_dict(event): + def event_dict(event: EventBase) -> JsonDict: d = event.get_dict() d.pop("redacted", None) d.pop("redacted_because", None) @@ -1477,18 +1474,20 @@ def event_dict(event): ), ) - def _store_rejected_events_txn(self, txn, events_and_contexts): + def _store_rejected_events_txn( + self, + txn: LoggingTransaction, + events_and_contexts: List[Tuple[EventBase, EventContext]], + ) -> List[Tuple[EventBase, EventContext]]: """Add rows to the 'rejections' table for received events which were rejected Args: - txn (twisted.enterprise.adbapi.Connection): db connection - events_and_contexts (list[(EventBase, EventContext)]): events - we are persisting + txn: db connection + events_and_contexts: events we are persisting Returns: - list[(EventBase, EventContext)] new list, without the rejected - events. + new list, without the rejected events. """ # Remove the rejected events from the list now that we've added them # to the events table and the events_json table. @@ -1509,7 +1508,7 @@ def _update_metadata_tables_txn( events_and_contexts: List[Tuple[EventBase, EventContext]], all_events_and_contexts: List[Tuple[EventBase, EventContext]], inhibit_local_membership_updates: bool = False, - ): + ) -> None: """Update all the miscellaneous tables for new events Args: @@ -1600,15 +1599,14 @@ def _update_metadata_tables_txn( inhibit_local_membership_updates=inhibit_local_membership_updates, ) - # Insert event_reference_hashes table. - self._store_event_reference_hashes_txn( - txn, [event for event, _ in events_and_contexts] - ) - # Prefill the event cache self._add_to_cache(txn, events_and_contexts) - def _add_to_cache(self, txn, events_and_contexts): + def _add_to_cache( + self, + txn: LoggingTransaction, + events_and_contexts: List[Tuple[EventBase, EventContext]], + ) -> None: to_prefill = [] rows = [] @@ -1639,7 +1637,7 @@ def _add_to_cache(self, txn, events_and_contexts): if not row["rejects"] and not row["redacts"]: to_prefill.append(EventCacheEntry(event=event, redacted_event=None)) - def prefill(): + def prefill() -> None: for cache_entry in to_prefill: self.store._get_event_cache.set( (cache_entry.event.event_id,), cache_entry @@ -1669,19 +1667,24 @@ def _store_redaction(self, txn: LoggingTransaction, event: EventBase) -> None: ) def insert_labels_for_event_txn( - self, txn, event_id, labels, room_id, topological_ordering - ): + self, + txn: LoggingTransaction, + event_id: str, + labels: List[str], + room_id: str, + topological_ordering: int, + ) -> None: """Store the mapping between an event's ID and its labels, with one row per (event_id, label) tuple. Args: - txn (LoggingTransaction): The transaction to execute. - event_id (str): The event's ID. - labels (list[str]): A list of text labels. - room_id (str): The ID of the room the event was sent to. - topological_ordering (int): The position of the event in the room's topology. + txn: The transaction to execute. + event_id: The event's ID. + labels: A list of text labels. + room_id: The ID of the room the event was sent to. + topological_ordering: The position of the event in the room's topology. """ - return self.db_pool.simple_insert_many_txn( + self.db_pool.simple_insert_many_txn( txn=txn, table="event_labels", keys=("event_id", "label", "room_id", "topological_ordering"), @@ -1690,44 +1693,32 @@ def insert_labels_for_event_txn( ], ) - def _insert_event_expiry_txn(self, txn, event_id, expiry_ts): + def _insert_event_expiry_txn( + self, txn: LoggingTransaction, event_id: str, expiry_ts: int + ) -> None: """Save the expiry timestamp associated with a given event ID. Args: - txn (LoggingTransaction): The database transaction to use. - event_id (str): The event ID the expiry timestamp is associated with. - expiry_ts (int): The timestamp at which to expire (delete) the event. + txn: The database transaction to use. + event_id: The event ID the expiry timestamp is associated with. + expiry_ts: The timestamp at which to expire (delete) the event. """ - return self.db_pool.simple_insert_txn( + self.db_pool.simple_insert_txn( txn=txn, table="event_expiry", values={"event_id": event_id, "expiry_ts": expiry_ts}, ) - def _store_event_reference_hashes_txn(self, txn, events): - """Store a hash for a PDU - Args: - txn (cursor): - events (list): list of Events. - """ - - vals = [] - for event in events: - ref_alg, ref_hash_bytes = compute_event_reference_hash(event) - vals.append((event.event_id, ref_alg, memoryview(ref_hash_bytes))) - - self.db_pool.simple_insert_many_txn( - txn, - table="event_reference_hashes", - keys=("event_id", "algorithm", "hash"), - values=vals, - ) - def _store_room_members_txn( - self, txn, events, *, inhibit_local_membership_updates: bool = False - ): + self, + txn: LoggingTransaction, + events: List[EventBase], + *, + inhibit_local_membership_updates: bool = False, + ) -> None: """ Store a room member in the database. + Args: txn: The transaction to use. events: List of events to store. @@ -1767,6 +1758,7 @@ def non_null_str_or_none(val: Any) -> Optional[str]: ) for event in events: + assert event.internal_metadata.stream_ordering is not None txn.call_after( self.store._membership_stream_cache.entity_has_changed, event.state_key, @@ -1863,7 +1855,9 @@ def _handle_event_relations( (parent_id, event.sender), ) - def _handle_insertion_event(self, txn: LoggingTransaction, event: EventBase): + def _handle_insertion_event( + self, txn: LoggingTransaction, event: EventBase + ) -> None: """Handles keeping track of insertion events and edges/connections. Part of MSC2716. @@ -1924,7 +1918,7 @@ def _handle_insertion_event(self, txn: LoggingTransaction, event: EventBase): }, ) - def _handle_batch_event(self, txn: LoggingTransaction, event: EventBase): + def _handle_batch_event(self, txn: LoggingTransaction, event: EventBase) -> None: """Handles inserting the batch edges/connections between the batch event and an insertion event. Part of MSC2716. @@ -2024,25 +2018,29 @@ def _handle_redact_relations( txn, table="event_relations", keyvalues={"event_id": redacted_event_id} ) - def _store_room_topic_txn(self, txn: LoggingTransaction, event: EventBase): + def _store_room_topic_txn(self, txn: LoggingTransaction, event: EventBase) -> None: if isinstance(event.content.get("topic"), str): self.store_event_search_txn( txn, event, "content.topic", event.content["topic"] ) - def _store_room_name_txn(self, txn: LoggingTransaction, event: EventBase): + def _store_room_name_txn(self, txn: LoggingTransaction, event: EventBase) -> None: if isinstance(event.content.get("name"), str): self.store_event_search_txn( txn, event, "content.name", event.content["name"] ) - def _store_room_message_txn(self, txn: LoggingTransaction, event: EventBase): + def _store_room_message_txn( + self, txn: LoggingTransaction, event: EventBase + ) -> None: if isinstance(event.content.get("body"), str): self.store_event_search_txn( txn, event, "content.body", event.content["body"] ) - def _store_retention_policy_for_room_txn(self, txn, event): + def _store_retention_policy_for_room_txn( + self, txn: LoggingTransaction, event: EventBase + ) -> None: if not event.is_state(): logger.debug("Ignoring non-state m.room.retention event") return @@ -2102,8 +2100,11 @@ def store_event_search_txn( ) def _set_push_actions_for_event_and_users_txn( - self, txn, events_and_contexts, all_events_and_contexts - ): + self, + txn: LoggingTransaction, + events_and_contexts: List[Tuple[EventBase, EventContext]], + all_events_and_contexts: List[Tuple[EventBase, EventContext]], + ) -> None: """Handles moving push actions from staging table to main event_push_actions table for all events in `events_and_contexts`. @@ -2111,12 +2112,10 @@ def _set_push_actions_for_event_and_users_txn( from the push action staging area. Args: - events_and_contexts (list[(EventBase, EventContext)]): events - we are persisting - all_events_and_contexts (list[(EventBase, EventContext)]): all - events that we were going to persist. This includes events - we've already persisted, etc, that wouldn't appear in - events_and_context. + events_and_contexts: events we are persisting + all_events_and_contexts: all events that we were going to persist. + This includes events we've already persisted, etc, that wouldn't + appear in events_and_context. """ # Only non outlier events will have push actions associated with them, @@ -2185,7 +2184,9 @@ def _set_push_actions_for_event_and_users_txn( ), ) - def _remove_push_actions_for_event_id_txn(self, txn, room_id, event_id): + def _remove_push_actions_for_event_id_txn( + self, txn: LoggingTransaction, room_id: str, event_id: str + ) -> None: # Sad that we have to blow away the cache for the whole room here txn.call_after( self.store.get_unread_event_push_actions_by_room_for_user.invalidate, @@ -2196,7 +2197,9 @@ def _remove_push_actions_for_event_id_txn(self, txn, room_id, event_id): (room_id, event_id), ) - def _store_rejections_txn(self, txn, event_id, reason): + def _store_rejections_txn( + self, txn: LoggingTransaction, event_id: str, reason: str + ) -> None: self.db_pool.simple_insert_txn( txn, table="rejections", @@ -2208,8 +2211,10 @@ def _store_rejections_txn(self, txn, event_id, reason): ) def _store_event_state_mappings_txn( - self, txn, events_and_contexts: Iterable[Tuple[EventBase, EventContext]] - ): + self, + txn: LoggingTransaction, + events_and_contexts: Collection[Tuple[EventBase, EventContext]], + ) -> None: state_groups = {} for event, context in events_and_contexts: if event.internal_metadata.is_outlier(): @@ -2266,7 +2271,9 @@ def _store_event_state_mappings_txn( state_group_id, ) - def _update_min_depth_for_room_txn(self, txn, room_id, depth): + def _update_min_depth_for_room_txn( + self, txn: LoggingTransaction, room_id: str, depth: int + ) -> None: min_depth = self.store._get_min_depth_interaction(txn, room_id) if min_depth is not None and depth >= min_depth: @@ -2279,7 +2286,9 @@ def _update_min_depth_for_room_txn(self, txn, room_id, depth): values={"min_depth": depth}, ) - def _handle_mult_prev_events(self, txn, events): + def _handle_mult_prev_events( + self, txn: LoggingTransaction, events: List[EventBase] + ) -> None: """ For the given event, update the event edges table and forward and backward extremities tables. @@ -2297,7 +2306,9 @@ def _handle_mult_prev_events(self, txn, events): self._update_backward_extremeties(txn, events) - def _update_backward_extremeties(self, txn, events): + def _update_backward_extremeties( + self, txn: LoggingTransaction, events: List[EventBase] + ) -> None: """Updates the event_backward_extremities tables based on the new/updated events being persisted. diff --git a/synapse/storage/databases/main/purge_events.py b/synapse/storage/databases/main/purge_events.py index bfc85b3add98..38ba91af4c47 100644 --- a/synapse/storage/databases/main/purge_events.py +++ b/synapse/storage/databases/main/purge_events.py @@ -69,7 +69,6 @@ def _purge_history_txn( # event_forward_extremities # event_json # event_push_actions - # event_reference_hashes # event_relations # event_search # event_to_state_groups @@ -220,7 +219,6 @@ def _purge_history_txn( "event_auth", "event_edges", "event_forward_extremities", - "event_reference_hashes", "event_relations", "event_search", "rejections", @@ -369,7 +367,6 @@ def _purge_room_txn(self, txn: LoggingTransaction, room_id: str) -> List[int]: "event_edges", "event_json", "event_push_actions_staging", - "event_reference_hashes", "event_relations", "event_to_state_groups", "event_auth_chains", diff --git a/synapse/storage/databases/main/search.py b/synapse/storage/databases/main/search.py index 3c49e7ec98e2..78e0773b2a88 100644 --- a/synapse/storage/databases/main/search.py +++ b/synapse/storage/databases/main/search.py @@ -14,7 +14,7 @@ import logging import re -from typing import TYPE_CHECKING, Any, Collection, Iterable, List, Optional, Set +from typing import TYPE_CHECKING, Any, Collection, Iterable, List, Optional, Set, Tuple import attr @@ -27,7 +27,7 @@ LoggingTransaction, ) from synapse.storage.databases.main.events_worker import EventRedactBehaviour -from synapse.storage.engines import PostgresEngine, Sqlite3Engine +from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine, Sqlite3Engine from synapse.types import JsonDict if TYPE_CHECKING: @@ -149,7 +149,9 @@ def __init__( self.EVENT_SEARCH_DELETE_NON_STRINGS, self._background_delete_non_strings ) - async def _background_reindex_search(self, progress, batch_size): + async def _background_reindex_search( + self, progress: JsonDict, batch_size: int + ) -> int: # we work through the events table from highest stream id to lowest target_min_stream_id = progress["target_min_stream_id_inclusive"] max_stream_id = progress["max_stream_id_exclusive"] @@ -157,7 +159,7 @@ async def _background_reindex_search(self, progress, batch_size): TYPES = ["m.room.name", "m.room.message", "m.room.topic"] - def reindex_search_txn(txn): + def reindex_search_txn(txn: LoggingTransaction) -> int: sql = ( "SELECT stream_ordering, event_id, room_id, type, json, " " origin_server_ts FROM events" @@ -255,12 +257,14 @@ def reindex_search_txn(txn): return result - async def _background_reindex_gin_search(self, progress, batch_size): + async def _background_reindex_gin_search( + self, progress: JsonDict, batch_size: int + ) -> int: """This handles old synapses which used GIST indexes, if any; converting them back to be GIN as per the actual schema. """ - def create_index(conn): + def create_index(conn: LoggingDatabaseConnection) -> None: conn.rollback() # we have to set autocommit, because postgres refuses to @@ -299,7 +303,9 @@ def create_index(conn): ) return 1 - async def _background_reindex_search_order(self, progress, batch_size): + async def _background_reindex_search_order( + self, progress: JsonDict, batch_size: int + ) -> int: target_min_stream_id = progress["target_min_stream_id_inclusive"] max_stream_id = progress["max_stream_id_exclusive"] rows_inserted = progress.get("rows_inserted", 0) @@ -307,7 +313,7 @@ async def _background_reindex_search_order(self, progress, batch_size): if not have_added_index: - def create_index(conn): + def create_index(conn: LoggingDatabaseConnection) -> None: conn.rollback() conn.set_session(autocommit=True) c = conn.cursor() @@ -336,7 +342,7 @@ def create_index(conn): pg, ) - def reindex_search_txn(txn): + def reindex_search_txn(txn: LoggingTransaction) -> Tuple[int, bool]: sql = ( "UPDATE event_search AS es SET stream_ordering = e.stream_ordering," " origin_server_ts = e.origin_server_ts" @@ -644,7 +650,8 @@ async def search_rooms( else: raise Exception("Unrecognized database engine") - args.append(limit) + # mypy expects to append only a `str`, not an `int` + args.append(limit) # type: ignore[arg-type] results = await self.db_pool.execute( "search_rooms", self.db_pool.cursor_to_dict, sql, *args @@ -705,7 +712,7 @@ async def _find_highlights_in_postgres( A set of strings. """ - def f(txn): + def f(txn: LoggingTransaction) -> Set[str]: highlight_words = set() for event in events: # As a hack we simply join values of all possible keys. This is @@ -759,11 +766,11 @@ def f(txn): return await self.db_pool.runInteraction("_find_highlights", f) -def _to_postgres_options(options_dict): +def _to_postgres_options(options_dict: JsonDict) -> str: return "'%s'" % (",".join("%s=%s" % (k, v) for k, v in options_dict.items()),) -def _parse_query(database_engine, search_term): +def _parse_query(database_engine: BaseDatabaseEngine, search_term: str) -> str: """Takes a plain unicode string from the user and converts it into a form that can be passed to database. We use this so that we can add prefix matching, which isn't something diff --git a/synapse/storage/persist_events.py b/synapse/storage/persist_events.py index 97118045a1ad..a7f6338e058d 100644 --- a/synapse/storage/persist_events.py +++ b/synapse/storage/persist_events.py @@ -487,12 +487,6 @@ async def _persist_event_batch( # extremities in each room new_forward_extremities: Dict[str, Set[str]] = {} - # map room_id->(type,state_key)->event_id tracking the full - # state in each room after adding these events. - # This is simply used to prefill the get_current_state_ids - # cache - current_state_for_room: Dict[str, StateMap[str]] = {} - # map room_id->(to_delete, to_insert) where to_delete is a list # of type/state keys to remove from current state, and to_insert # is a map (type,key)->event_id giving the state delta in each @@ -628,14 +622,8 @@ async def _persist_event_batch( state_delta_for_room[room_id] = delta - # If we have the current_state then lets prefill - # the cache with it. - if current_state is not None: - current_state_for_room[room_id] = current_state - await self.persist_events_store._persist_events_and_state_updates( chunk, - current_state_for_room=current_state_for_room, state_delta_for_room=state_delta_for_room, new_forward_extremities=new_forward_extremities, use_negative_stream_ordering=backfilled, @@ -733,7 +721,8 @@ async def _get_new_state_after_events( The first state map is the full new current state and the second is the delta to the existing current state. If both are None then - there has been no change. + there has been no change. Either or neither can be None if there + has been a change. The function may prune some old entries from the set of new forward extremities if it's safe to do so. @@ -743,9 +732,6 @@ async def _get_new_state_after_events( the new current state is only returned if we've already calculated it. """ - # map from state_group to ((type, key) -> event_id) state map - state_groups_map = {} - # Map from (prev state group, new state group) -> delta state dict state_group_deltas = {} @@ -759,16 +745,6 @@ async def _get_new_state_after_events( ) continue - if ctx.state_group in state_groups_map: - continue - - # We're only interested in pulling out state that has already - # been cached in the context. We'll pull stuff out of the DB later - # if necessary. - current_state_ids = ctx.get_cached_current_state_ids() - if current_state_ids is not None: - state_groups_map[ctx.state_group] = current_state_ids - if ctx.prev_group: state_group_deltas[(ctx.prev_group, ctx.state_group)] = ctx.delta_ids @@ -826,18 +802,14 @@ async def _get_new_state_after_events( delta_ids = state_group_deltas.get((old_state_group, new_state_group), None) if delta_ids is not None: # We have a delta from the existing to new current state, - # so lets just return that. If we happen to already have - # the current state in memory then lets also return that, - # but it doesn't matter if we don't. - new_state = state_groups_map.get(new_state_group) - return new_state, delta_ids, new_latest_event_ids + # so lets just return that. + return None, delta_ids, new_latest_event_ids # Now that we have calculated new_state_groups we need to get # their state IDs so we can resolve to a single state set. - missing_state = new_state_groups - set(state_groups_map) - if missing_state: - group_to_state = await self.state_store._get_state_for_groups(missing_state) - state_groups_map.update(group_to_state) + state_groups_map = await self.state_store._get_state_for_groups( + new_state_groups + ) if len(new_state_groups) == 1: # If there is only one state group, then we know what the current diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index 871d4ace123c..20c344faeab3 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -SCHEMA_VERSION = 69 # remember to update the list below when updating +SCHEMA_VERSION = 70 # remember to update the list below when updating """Represents the expectations made by the codebase about the database schema This should be incremented whenever the codebase changes its requirements on the @@ -62,6 +62,9 @@ Changes in SCHEMA_VERSION = 69: - We now write to `device_lists_changes_in_room` table. - Use sequence to generate future `application_services_txns.txn_id`s + +Changes in SCHEMA_VERSION = 70: + - event_reference_hashes is no longer written to. """ diff --git a/tests/config/test_workers.py b/tests/config/test_workers.py index da81bb9655fc..ef6294ecb2ce 100644 --- a/tests/config/test_workers.py +++ b/tests/config/test_workers.py @@ -286,3 +286,30 @@ def test_new_configs_appservice_worker(self) -> None: "notify_appservices_from_worker", ) ) + + def test_worker_duty_configs(self) -> None: + """ + Additional tests for the worker duties + """ + + worker1_config = self._make_worker_config( + worker_app="synapse.app.generic_worker", + worker_name="worker1", + extras={ + "notify_appservices_from_worker": "worker2", + "update_user_directory_from_worker": "worker1", + }, + ) + self.assertFalse(worker1_config.should_notify_appservices) + self.assertTrue(worker1_config.should_update_user_directory) + + worker2_config = self._make_worker_config( + worker_app="synapse.app.generic_worker", + worker_name="worker2", + extras={ + "notify_appservices_from_worker": "worker2", + "update_user_directory_from_worker": "worker1", + }, + ) + self.assertTrue(worker2_config.should_notify_appservices) + self.assertFalse(worker2_config.should_update_user_directory) diff --git a/tests/handlers/test_federation_event.py b/tests/handlers/test_federation_event.py index 489ba5773672..e64b28f28b86 100644 --- a/tests/handlers/test_federation_event.py +++ b/tests/handlers/test_federation_event.py @@ -148,7 +148,9 @@ def _test_process_pulled_event_with_missing_state( prev_event.internal_metadata.outlier = True persistence = self.hs.get_storage().persistence self.get_success( - persistence.persist_event(prev_event, EventContext.for_outlier()) + persistence.persist_event( + prev_event, EventContext.for_outlier(self.hs.get_storage()) + ) ) else: diff --git a/tests/http/server/__init__.py b/tests/http/server/__init__.py new file mode 100644 index 000000000000..3a5f22c02235 --- /dev/null +++ b/tests/http/server/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/http/server/_base.py b/tests/http/server/_base.py new file mode 100644 index 000000000000..b9f1a381aa2b --- /dev/null +++ b/tests/http/server/_base.py @@ -0,0 +1,100 @@ +# Copyright 2022 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unles4s required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from http import HTTPStatus +from typing import Any, Callable, Optional, Union +from unittest import mock + +from twisted.internet.error import ConnectionDone + +from synapse.http.server import ( + HTTP_STATUS_REQUEST_CANCELLED, + respond_with_html_bytes, + respond_with_json, +) +from synapse.types import JsonDict + +from tests import unittest +from tests.server import FakeChannel, ThreadedMemoryReactorClock + + +class EndpointCancellationTestHelperMixin(unittest.TestCase): + """Provides helper methods for testing cancellation of endpoints.""" + + def _test_disconnect( + self, + reactor: ThreadedMemoryReactorClock, + channel: FakeChannel, + expect_cancellation: bool, + expected_body: Union[bytes, JsonDict], + expected_code: Optional[int] = None, + ) -> None: + """Disconnects an in-flight request and checks the response. + + Args: + reactor: The twisted reactor running the request handler. + channel: The `FakeChannel` for the request. + expect_cancellation: `True` if request processing is expected to be + cancelled, `False` if the request should run to completion. + expected_body: The expected response for the request. + expected_code: The expected status code for the request. Defaults to `200` + or `499` depending on `expect_cancellation`. + """ + # Determine the expected status code. + if expected_code is None: + if expect_cancellation: + expected_code = HTTP_STATUS_REQUEST_CANCELLED + else: + expected_code = HTTPStatus.OK + + request = channel.request + self.assertFalse( + channel.is_finished(), + "Request finished before we could disconnect - " + "was `await_result=False` passed to `make_request`?", + ) + + # We're about to disconnect the request. This also disconnects the channel, so + # we have to rely on mocks to extract the response. + respond_method: Callable[..., Any] + if isinstance(expected_body, bytes): + respond_method = respond_with_html_bytes + else: + respond_method = respond_with_json + + with mock.patch( + f"synapse.http.server.{respond_method.__name__}", wraps=respond_method + ) as respond_mock: + # Disconnect the request. + request.connectionLost(reason=ConnectionDone()) + + if expect_cancellation: + # An immediate cancellation is expected. + respond_mock.assert_called_once() + args, _kwargs = respond_mock.call_args + code, body = args[1], args[2] + self.assertEqual(code, expected_code) + self.assertEqual(request.code, expected_code) + self.assertEqual(body, expected_body) + else: + respond_mock.assert_not_called() + + # The handler is expected to run to completion. + reactor.pump([1.0]) + respond_mock.assert_called_once() + args, _kwargs = respond_mock.call_args + code, body = args[1], args[2] + self.assertEqual(code, expected_code) + self.assertEqual(request.code, expected_code) + self.assertEqual(body, expected_body) diff --git a/tests/server.py b/tests/server.py index 8f30e250c83c..aaefcfc46cd9 100644 --- a/tests/server.py +++ b/tests/server.py @@ -109,6 +109,17 @@ class FakeChannel: _ip: str = "127.0.0.1" _producer: Optional[Union[IPullProducer, IPushProducer]] = None resource_usage: Optional[ContextResourceUsage] = None + _request: Optional[Request] = None + + @property + def request(self) -> Request: + assert self._request is not None + return self._request + + @request.setter + def request(self, request: Request) -> None: + assert self._request is None + self._request = request @property def json_body(self): @@ -322,6 +333,8 @@ def make_request( channel = FakeChannel(site, reactor, ip=client_ip) req = request(channel, site) + channel.request = req + req.content = BytesIO(content) # Twisted expects to be at the end of the content when parsing the request. req.content.seek(0, SEEK_END) diff --git a/tests/storage/test_event_chain.py b/tests/storage/test_event_chain.py index 401020fd6361..c7661e71868f 100644 --- a/tests/storage/test_event_chain.py +++ b/tests/storage/test_event_chain.py @@ -393,7 +393,7 @@ def _persist(txn): # We need to persist the events to the events and state_events # tables. persist_events_store._store_event_txn( - txn, [(e, EventContext()) for e in events] + txn, [(e, EventContext(self.hs.get_storage())) for e in events] ) # Actually call the function that calculates the auth chain stuff. diff --git a/tests/storage/test_event_federation.py b/tests/storage/test_event_federation.py index 645d564d1c40..d92a9ac5b798 100644 --- a/tests/storage/test_event_federation.py +++ b/tests/storage/test_event_federation.py @@ -58,15 +58,6 @@ def insert_event(txn, i): (room_id, event_id), ) - txn.execute( - ( - "INSERT INTO event_reference_hashes " - "(event_id, algorithm, hash) " - "VALUES (?, 'sha256', ?)" - ), - (event_id, bytearray(b"ffff")), - ) - for i in range(0, 20): self.get_success( self.store.db_pool.runInteraction("insert", insert_event, i) diff --git a/tests/test_state.py b/tests/test_state.py index e4baa6913746..651ec1c7d4bd 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -88,6 +88,9 @@ async def get_state_groups_ids(self, room_id, event_ids): return groups + async def get_state_ids_for_group(self, state_group): + return self._group_to_state[state_group] + async def store_state_group( self, event_id, room_id, prev_group, delta_ids, current_state_ids ): diff --git a/tests/test_visibility.py b/tests/test_visibility.py index d0230f9ebbc5..7a9b01ef9d44 100644 --- a/tests/test_visibility.py +++ b/tests/test_visibility.py @@ -234,7 +234,9 @@ def _inject_outlier(self) -> EventBase: event = self.get_success(builder.build(prev_event_ids=[], auth_event_ids=[])) event.internal_metadata.outlier = True self.get_success( - self.storage.persistence.persist_event(event, EventContext.for_outlier()) + self.storage.persistence.persist_event( + event, EventContext.for_outlier(self.storage) + ) ) return event