From 4216dc6d78828f924e5b10b876cad472d1b75eb5 Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Mon, 22 Jul 2019 20:55:22 +1000 Subject: [PATCH 01/12] put back these options --- changelog.d/5729.bugfix | 1 + synapse/config/workers.py | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 changelog.d/5729.bugfix diff --git a/changelog.d/5729.bugfix b/changelog.d/5729.bugfix new file mode 100644 index 000000000000..8817a3e84096 --- /dev/null +++ b/changelog.d/5729.bugfix @@ -0,0 +1 @@ +Fix worker log config not being set correctly. diff --git a/synapse/config/workers.py b/synapse/config/workers.py index 246d72cd611b..21e6259d60fe 100644 --- a/synapse/config/workers.py +++ b/synapse/config/workers.py @@ -31,6 +31,7 @@ def read_config(self, config, **kwargs): self.worker_listeners = config.get("worker_listeners", []) self.worker_daemonize = config.get("worker_daemonize") self.worker_pid_file = config.get("worker_pid_file") + self.worker_log_config = config.get("worker_log_config") # The host used to connect to the main synapse self.worker_replication_host = config.get("worker_replication_host", None) @@ -78,3 +79,5 @@ def read_arguments(self, args): self.worker_daemonize = args.daemonize if args.manhole is not None: self.worker_manhole = args.worker_manhole + if args.log_config is not None: + self.worker_log_config = args.log_config From a27335310e0fc873cdace885ef41996d0d19b1ca Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Mon, 22 Jul 2019 21:08:43 +1000 Subject: [PATCH 02/12] reverted too much --- synapse/config/workers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/config/workers.py b/synapse/config/workers.py index 21e6259d60fe..bc0fc165e3a1 100644 --- a/synapse/config/workers.py +++ b/synapse/config/workers.py @@ -79,5 +79,3 @@ def read_arguments(self, args): self.worker_daemonize = args.daemonize if args.manhole is not None: self.worker_manhole = args.worker_manhole - if args.log_config is not None: - self.worker_log_config = args.log_config From aeff36a770c3750d24a5b7c320ccdaeebc03c4bd Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Mon, 22 Jul 2019 21:34:17 +1000 Subject: [PATCH 03/12] use custom blacklist for workers --- .buildkite/pipeline.yml | 3 ++- worker-blacklist | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 worker-blacklist diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index d5e5aeec6b84..6171089ca280 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -220,8 +220,10 @@ steps: env: POSTGRES: "1" WORKERS: "1" + BLACKLIST: "synapse-blacklist-with-workers" command: - "bash .buildkite/merge_base_branch.sh" + - "bash -c 'cat synapse-blacklist worker-blacklist > synapse-blacklist-with-workers'" - "bash /synapse_sytest.sh" plugins: - docker#v3.0.1: @@ -229,7 +231,6 @@ steps: propagate-environment: true always-pull: true workdir: "/src" - soft_fail: true retry: automatic: - exit_status: -1 diff --git a/worker-blacklist b/worker-blacklist new file mode 100644 index 000000000000..c34ed54833d6 --- /dev/null +++ b/worker-blacklist @@ -0,0 +1,14 @@ +# This file serves as a blacklist for SyTest tests that we expect will fail in +# Synapse when run under worker mode. For more details, see sytest-blacklist. + +Message history can be paginated + +m.room.history_visibility == "world_readable" allows/forbids appropriately for Real users + +Can re-join room if re-invited + +/upgrade creates a new room + +The only membership state included in an initial sync is for all the senders in the timeline + +Local device key changes get to remote servers \ No newline at end of file From 5b04ca36f5f48eadaee21a9237f310ad8c01b65f Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Mon, 22 Jul 2019 21:35:35 +1000 Subject: [PATCH 04/12] fix --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 834ddfad39c6..235f0f973e59 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,6 +8,7 @@ include demo/demo.tls.dh include demo/*.py include demo/*.sh include sytest-blacklist +include worker-blacklist recursive-include synapse/storage/schema *.sql recursive-include synapse/storage/schema *.sql.postgres From 584f272a60da3f275bafb8cd38b07f40a5ef1d44 Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Mon, 22 Jul 2019 22:04:23 +1000 Subject: [PATCH 05/12] fix --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 6171089ca280..48a3e6b21c52 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -223,7 +223,7 @@ steps: BLACKLIST: "synapse-blacklist-with-workers" command: - "bash .buildkite/merge_base_branch.sh" - - "bash -c 'cat synapse-blacklist worker-blacklist > synapse-blacklist-with-workers'" + - "bash -c 'cat /src/synapse-blacklist /src/worker-blacklist > /src/synapse-blacklist-with-workers'" - "bash /synapse_sytest.sh" plugins: - docker#v3.0.1: From bf69536e6303e62e72454dbc824cda78e035a647 Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Mon, 22 Jul 2019 22:09:57 +1000 Subject: [PATCH 06/12] fix --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 48a3e6b21c52..80dc4c67321b 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -223,7 +223,7 @@ steps: BLACKLIST: "synapse-blacklist-with-workers" command: - "bash .buildkite/merge_base_branch.sh" - - "bash -c 'cat /src/synapse-blacklist /src/worker-blacklist > /src/synapse-blacklist-with-workers'" + - "bash -c 'cat /src/sytest-blacklist /src/worker-blacklist > /src/synapse-blacklist-with-workers'" - "bash /synapse_sytest.sh" plugins: - docker#v3.0.1: From 8081512a7bd064f3bf33e0ea1c960f2105ce23fc Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Mon, 22 Jul 2019 22:27:05 +1000 Subject: [PATCH 07/12] update blacklist --- worker-blacklist | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/worker-blacklist b/worker-blacklist index c34ed54833d6..c434439652d1 100644 --- a/worker-blacklist +++ b/worker-blacklist @@ -11,4 +11,8 @@ Can re-join room if re-invited The only membership state included in an initial sync is for all the senders in the timeline -Local device key changes get to remote servers \ No newline at end of file +Local device key changes get to remote servers + +If remote user leaves room we no longer receive device updates + +Forgotten room messages cannot be paginated \ No newline at end of file From 40dfb35711432e0d57de505d95837606e0ba9aff Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Mon, 22 Jul 2019 22:44:14 +1000 Subject: [PATCH 08/12] update blacklist --- worker-blacklist | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/worker-blacklist b/worker-blacklist index c434439652d1..8f1355ea7653 100644 --- a/worker-blacklist +++ b/worker-blacklist @@ -3,6 +3,8 @@ Message history can be paginated +m.room.history_visibility == "world_readable" allows/forbids appropriately for Guest users + m.room.history_visibility == "world_readable" allows/forbids appropriately for Real users Can re-join room if re-invited @@ -15,4 +17,10 @@ Local device key changes get to remote servers If remote user leaves room we no longer receive device updates -Forgotten room messages cannot be paginated \ No newline at end of file +Forgotten room messages cannot be paginated + +Inbound federation can get public room list + +Members from the gap are included in gappy incr LL sync + +Leaves are present in non-gapped incremental syncs \ No newline at end of file From 63c9e8777c008da04ed380bba29a432b94cb74c4 Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Mon, 22 Jul 2019 23:18:44 +1000 Subject: [PATCH 09/12] merge changelog --- changelog.d/5729.bugfix | 1 - changelog.d/5729.removal | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 changelog.d/5729.bugfix create mode 100644 changelog.d/5729.removal diff --git a/changelog.d/5729.bugfix b/changelog.d/5729.bugfix deleted file mode 100644 index 8817a3e84096..000000000000 --- a/changelog.d/5729.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix worker log config not being set correctly. diff --git a/changelog.d/5729.removal b/changelog.d/5729.removal new file mode 100644 index 000000000000..3af5198e6bb4 --- /dev/null +++ b/changelog.d/5729.removal @@ -0,0 +1 @@ + Synapse now no longer accepts the `-v`/`--verbose`, `-f`/`--log-file`, or `--log-config` command line flags, and removes the deprecated `verbose` and `log_file` configuration file options. Users of these options should migrate their options into the dedicated log configuration. From 57c5c9b96d249a5d665511a86370b8f9d69311af Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Mon, 22 Jul 2019 23:20:21 +1000 Subject: [PATCH 10/12] move some things around --- .buildkite/pipeline.yml | 2 +- worker-blacklist => .buildkite/worker-blacklist | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename worker-blacklist => .buildkite/worker-blacklist (100%) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 80dc4c67321b..c8ae1a44bed5 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -223,7 +223,7 @@ steps: BLACKLIST: "synapse-blacklist-with-workers" command: - "bash .buildkite/merge_base_branch.sh" - - "bash -c 'cat /src/sytest-blacklist /src/worker-blacklist > /src/synapse-blacklist-with-workers'" + - "bash -c 'cat /src/sytest-blacklist /src/.buildkite/worker-blacklist > /src/synapse-blacklist-with-workers'" - "bash /synapse_sytest.sh" plugins: - docker#v3.0.1: diff --git a/worker-blacklist b/.buildkite/worker-blacklist similarity index 100% rename from worker-blacklist rename to .buildkite/worker-blacklist From 4f3a96cc6d7a8b845fff46910b814043cb59f2aa Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Mon, 22 Jul 2019 23:32:37 +1000 Subject: [PATCH 11/12] more failing tests --- .buildkite/worker-blacklist | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.buildkite/worker-blacklist b/.buildkite/worker-blacklist index 8f1355ea7653..a211ed7b1874 100644 --- a/.buildkite/worker-blacklist +++ b/.buildkite/worker-blacklist @@ -23,4 +23,6 @@ Inbound federation can get public room list Members from the gap are included in gappy incr LL sync -Leaves are present in non-gapped incremental syncs \ No newline at end of file +Leaves are present in non-gapped incremental syncs + +Old leaves are present in gapped incremental syncs \ No newline at end of file From 2806640c091f9aa54ee42fa4f94540f9ac91f308 Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Tue, 23 Jul 2019 00:48:36 +1000 Subject: [PATCH 12/12] fix --- MANIFEST.in | 1 - 1 file changed, 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 235f0f973e59..834ddfad39c6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,7 +8,6 @@ include demo/demo.tls.dh include demo/*.py include demo/*.sh include sytest-blacklist -include worker-blacklist recursive-include synapse/storage/schema *.sql recursive-include synapse/storage/schema *.sql.postgres